fix rss push bugs when < 100, improved code structure
This commit is contained in:
parent
2d8b314e32
commit
fcc11e8ce4
34
main.py
34
main.py
@ -5,11 +5,14 @@ from config import __CONFIG__
|
|||||||
import mysql_connector
|
import mysql_connector
|
||||||
from fastgpt_uploader import upload2fastgpt
|
from fastgpt_uploader import upload2fastgpt
|
||||||
from semanticscholar import search_paper
|
from semanticscholar import search_paper
|
||||||
from rss import load_rss
|
import rss
|
||||||
|
|
||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
|
|
||||||
def query(query:str):
|
@app.get("/fastdoi")
|
||||||
|
async def get_reference(questions):
|
||||||
|
print('Search: '+questions)
|
||||||
|
questions = urllib.parse.quote(questions)
|
||||||
res = []
|
res = []
|
||||||
try:
|
try:
|
||||||
list = search_paper(query)
|
list = search_paper(query)
|
||||||
@ -27,20 +30,31 @@ def query(query:str):
|
|||||||
print('New load: '+str(len(res))+'/'+str(len(list)))
|
print('New load: '+str(len(res))+'/'+str(len(list)))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(str(e))
|
print(str(e))
|
||||||
return res
|
|
||||||
|
|
||||||
@app.get("/fastdoi")
|
|
||||||
async def get_reference(questions):
|
|
||||||
print('Search: '+questions)
|
|
||||||
res = query(urllib.parse.quote(questions))
|
|
||||||
if(upload2fastgpt(res)):
|
if(upload2fastgpt(res)):
|
||||||
for i in res:
|
for i in res:
|
||||||
mysql_connector.new_load(i['id'])
|
mysql_connector.new_load(i['id'])
|
||||||
return res
|
return res
|
||||||
|
|
||||||
@app.get("/rss")
|
@app.get("/rss")
|
||||||
async def miniflux_rss():
|
async def load_rss():
|
||||||
return load_rss()
|
count = 0
|
||||||
|
loaded = []
|
||||||
|
uploads = []
|
||||||
|
for entry in rss.load_content():
|
||||||
|
loaded.append(entry['id'])
|
||||||
|
uploads.append({
|
||||||
|
'q': entry['title'],
|
||||||
|
'a': entry['content']
|
||||||
|
})
|
||||||
|
count += 1
|
||||||
|
if len(loaded) >= 100 and upload2fastgpt(uploads):
|
||||||
|
rss.update(loaded)
|
||||||
|
loaded = []
|
||||||
|
uploads = []
|
||||||
|
if len(loaded) > 0 and upload2fastgpt(uploads):
|
||||||
|
rss.update(loaded)
|
||||||
|
print(f"Total entries loaded: {count}")
|
||||||
|
return f"Total entries loaded: {count}"
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
import uvicorn
|
import uvicorn
|
||||||
|
24
rss.py
24
rss.py
@ -2,13 +2,10 @@ import miniflux
|
|||||||
import time
|
import time
|
||||||
|
|
||||||
from config import __CONFIG__
|
from config import __CONFIG__
|
||||||
from fastgpt_uploader import upload2fastgpt
|
|
||||||
|
|
||||||
# Authentication using an API token
|
# Authentication using an API token
|
||||||
client = miniflux.Client(__CONFIG__['miniflux_host'], api_key=__CONFIG__['miniflux_api'])
|
client = miniflux.Client(__CONFIG__['miniflux_host'], api_key=__CONFIG__['miniflux_api'])
|
||||||
|
|
||||||
count = 0
|
|
||||||
|
|
||||||
def load_content():
|
def load_content():
|
||||||
for i in range(1,10):
|
for i in range(1,10):
|
||||||
try:
|
try:
|
||||||
@ -26,26 +23,7 @@ def update(ids):
|
|||||||
client.update_entries(entry_ids=ids, status='read')
|
client.update_entries(entry_ids=ids, status='read')
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error uploading:{count} {e}")
|
print(f"Error updating: {e}")
|
||||||
time.sleep(60)
|
time.sleep(60)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def load_rss():
|
|
||||||
count = 0
|
|
||||||
loaded = []
|
|
||||||
uploads = []
|
|
||||||
for entry in load_content():
|
|
||||||
loaded.append(entry['id'])
|
|
||||||
uploads.append({
|
|
||||||
'q': entry['title'],
|
|
||||||
'a': entry['content']
|
|
||||||
})
|
|
||||||
count += 1
|
|
||||||
if len(loaded) >= 100 and upload2fastgpt(uploads):
|
|
||||||
update(loaded)
|
|
||||||
loaded = []
|
|
||||||
uploads = []
|
|
||||||
if len(loaded) > 0:
|
|
||||||
update(loaded)
|
|
||||||
print(f"Total entries loaded: {count}")
|
|
||||||
return f"Total entries loaded: {count}"
|
|
||||||
|
Loading…
Reference in New Issue
Block a user