Attachment 225020 Details for Bug 255405 – Updated cold start script.

Updated cold start script.

moin-coldstart.py (text/plain), 3.80 KB, created by Jethro Nederhof on 2021-05-17 10:14:26 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Jethro Nederhof

Created: 2021-05-17 10:14:26 UTC

Size: 3.80 KB

patch

obsolete

>#!/usr/bin/env python2.7
>from __future__ import print_function
>from collections import Counter, defaultdict
>from csv import reader
>from datetime import datetime, timedelta
>import gc
>from os import makedirs
>from os.path import exists, join
>from pickle import dump, HIGHEST_PROTOCOL
>from urllib import unquote_plus
>from sys import argv
>from time import mktime
>
>def main(datadir, write_caches=False):
>    totals = {}
>
>    watermark = mktime((datetime.utcnow() - timedelta(days=90)).timetuple())
>    # setup
>    views = defaultdict(int)
>    saves = defaultdict(int)
>    ignore = Counter()
>    latest = watermark
>    active_date = None
>    with open(join(datadir, "event-log"), "rb") as f:
>        csvr = reader(f, delimiter="\t", lineterminator="\n")
>        for row in csvr:
>            try:
>                tstamp, eventtype, values = row
>            except ValueError:
>                continue
>
>            if eventtype not in ("VIEWPAGE", "SAVEPAGE"):
>                continue
>
>            tstamp = long(tstamp) / 1000000.0 
>            if tstamp < latest:
>                continue
>            latest = max(latest, tstamp)
>            row_date = datetime.fromtimestamp(tstamp).strftime("%Y-%m-%d")
>            # new date
>            if active_date != row_date:
>                for pagename in totals:
>                    d = active_date
>                    v = views.get(pagename, 0)
>                    s = saves.get(pagename, 0)
>
>                    totals[pagename].append((d, v, s))
>
>                active_date = row_date
>                views = defaultdict(int)
>                saves = defaultdict(int)
>                print("processing:", active_date, "pages so far:", len(totals))
>                gc.collect()
>
>            params = dict([pair.split("=", 1) for pair in values.split("&")])
>            pagename = unquote_plus(params.get("pagename", ""))
>            if not pagename:
>                continue
>            if pagename in ignore:
>                ignore.update((pagename,))
>                continue
>
>            # start counting this page from this date
>            if pagename not in totals:
>                if exists(join(datadir, "pages", pagename)) or exists(join(datadir, "..", "underlay", "pages", pagename)):
>                    totals[pagename] = []
>                else:
>                    ignore.update((pagename,))
>
>            if eventtype == "VIEWPAGE":
>                views[pagename] += 1
>            else: # SAVEPAGE
>                saves[pagename] += 1
>
>    # process last date
>    for pagename in totals:
>        d = active_date
>        v = views.get(pagename, 0)
>        s = saves.get(pagename, 0)
>
>        totals[pagename].append((d, v, s))
>
>    for page, data in totals.iteritems():
>        # only produce cache for existing pages
>        page_path = join(datadir, "pages", page)
>        if not exists(page_path):
>            continue
>
>        cache_path = join(page_path, "cache")
>        try:
>            makedirs(cache_path)
>        except:
>            pass
>
>        records = [latest] + [list(x) for x in zip(*data)]
>        if write_caches:
>            with open(join(cache_path, "hitcounts"), "wb") as cache:
>                dump(tuple(records), cache, HIGHEST_PROTOCOL)
>        else:
>            # dry run
>            print (
>                "%s: update ts(%f) latest date(%s) latest views(%d) latest saves(%d)" %
>                (join(cache_path, "hitcounts"), records[0], records[1][-1], records[2][-1], records[3][-1])
>            )
>    
>    print("Done.")
>    if ignore:
>        print("Most common not found pages:")
>        print(ignore.most_common(100))
>
>
>if __name__ == "__main__":
>    assert len(argv) >= 2, "usage: %s MOIN_DATA_DIR [-w]" % argv[0]
>    # dry run unless -w specified
>    main(argv[1], "-w" in argv)
>

Actions: View

Attachments on bug 255405: 224440 | 225020