diff options
| -rwxr-xr-x | unbiased/main.py | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/unbiased/main.py b/unbiased/main.py index f784bce..c8a113e 100755 --- a/unbiased/main.py +++ b/unbiased/main.py @@ -22,11 +22,17 @@ def main(): parser.add_argument('-s', '--scratch', default='/opt/unbiased/scratch', help='writable scratch workspace') args = parser.parse_args() + crawl_frequency = 600 while True: logger.info('Starting crawl') + start = time.time() run(args.webroot, args.scratch) - logger.info('Crawl complete. Sleeping for 600s') - time.sleep(600) + finish = time.time() + runtime = finish - start + sleeptime = crawl_frequency - runtime + logger.info('Crawl complete in {}s. Sleeping for {}s'.format(int(runtime), int(sleeptime))) + if sleeptime > 0: + time.sleep(sleeptime) def run(webroot, scratch): sourceList=[] |