fix rss push bugs when < 100, improved code structure
This commit is contained in:
parent
2d8b314e32
commit
fcc11e8ce4
34
main.py
34
main.py
@ -5,11 +5,14 @@ from config import __CONFIG__
|
||||
import mysql_connector
|
||||
from fastgpt_uploader import upload2fastgpt
|
||||
from semanticscholar import search_paper
|
||||
from rss import load_rss
|
||||
import rss
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
def query(query:str):
|
||||
@app.get("/fastdoi")
|
||||
async def get_reference(questions):
|
||||
print('Search: '+questions)
|
||||
questions = urllib.parse.quote(questions)
|
||||
res = []
|
||||
try:
|
||||
list = search_paper(query)
|
||||
@ -27,20 +30,31 @@ def query(query:str):
|
||||
print('New load: '+str(len(res))+'/'+str(len(list)))
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
return res
|
||||
|
||||
@app.get("/fastdoi")
|
||||
async def get_reference(questions):
|
||||
print('Search: '+questions)
|
||||
res = query(urllib.parse.quote(questions))
|
||||
if(upload2fastgpt(res)):
|
||||
for i in res:
|
||||
mysql_connector.new_load(i['id'])
|
||||
return res
|
||||
|
||||
@app.get("/rss")
|
||||
async def miniflux_rss():
|
||||
return load_rss()
|
||||
async def load_rss():
|
||||
count = 0
|
||||
loaded = []
|
||||
uploads = []
|
||||
for entry in rss.load_content():
|
||||
loaded.append(entry['id'])
|
||||
uploads.append({
|
||||
'q': entry['title'],
|
||||
'a': entry['content']
|
||||
})
|
||||
count += 1
|
||||
if len(loaded) >= 100 and upload2fastgpt(uploads):
|
||||
rss.update(loaded)
|
||||
loaded = []
|
||||
uploads = []
|
||||
if len(loaded) > 0 and upload2fastgpt(uploads):
|
||||
rss.update(loaded)
|
||||
print(f"Total entries loaded: {count}")
|
||||
return f"Total entries loaded: {count}"
|
||||
|
||||
if __name__ == '__main__':
|
||||
import uvicorn
|
||||
|
24
rss.py
24
rss.py
@ -2,13 +2,10 @@ import miniflux
|
||||
import time
|
||||
|
||||
from config import __CONFIG__
|
||||
from fastgpt_uploader import upload2fastgpt
|
||||
|
||||
# Authentication using an API token
|
||||
client = miniflux.Client(__CONFIG__['miniflux_host'], api_key=__CONFIG__['miniflux_api'])
|
||||
|
||||
count = 0
|
||||
|
||||
def load_content():
|
||||
for i in range(1,10):
|
||||
try:
|
||||
@ -26,26 +23,7 @@ def update(ids):
|
||||
client.update_entries(entry_ids=ids, status='read')
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error uploading:{count} {e}")
|
||||
print(f"Error updating: {e}")
|
||||
time.sleep(60)
|
||||
return False
|
||||
|
||||
def load_rss():
|
||||
count = 0
|
||||
loaded = []
|
||||
uploads = []
|
||||
for entry in load_content():
|
||||
loaded.append(entry['id'])
|
||||
uploads.append({
|
||||
'q': entry['title'],
|
||||
'a': entry['content']
|
||||
})
|
||||
count += 1
|
||||
if len(loaded) >= 100 and upload2fastgpt(uploads):
|
||||
update(loaded)
|
||||
loaded = []
|
||||
uploads = []
|
||||
if len(loaded) > 0:
|
||||
update(loaded)
|
||||
print(f"Total entries loaded: {count}")
|
||||
return f"Total entries loaded: {count}"
|
||||
|
Loading…
Reference in New Issue
Block a user