summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xunbiased/main.py10
1 files changed, 8 insertions, 2 deletions
diff --git a/unbiased/main.py b/unbiased/main.py
index f784bce..c8a113e 100755
--- a/unbiased/main.py
+++ b/unbiased/main.py
@@ -22,11 +22,17 @@ def main():
parser.add_argument('-s', '--scratch', default='/opt/unbiased/scratch', help='writable scratch workspace')
args = parser.parse_args()
+ crawl_frequency = 600
while True:
logger.info('Starting crawl')
+ start = time.time()
run(args.webroot, args.scratch)
- logger.info('Crawl complete. Sleeping for 600s')
- time.sleep(600)
+ finish = time.time()
+ runtime = finish - start
+ sleeptime = crawl_frequency - runtime
+ logger.info('Crawl complete in {}s. Sleeping for {}s'.format(int(runtime), int(sleeptime)))
+ if sleeptime > 0:
+ time.sleep(sleeptime)
def run(webroot, scratch):
sourceList=[]