1.1 --- a/juupdater.py Fri Jul 24 02:37:53 2009 +0900
1.2 +++ b/juupdater.py Sat Jul 25 00:59:05 2009 +0900
1.3 @@ -7,105 +7,111 @@
1.4
1.5 from lib.jubeatdb import *
1.6
1.7 +THRESHOLD = 86400
1.8 +RETRYCOUNT = 3
1.9 +forcebefore = int(time.time()) - THRESHOLD
1.10 +
1.11 def log(s):
1.12 sys.stdout.write('%s | %s\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), s))
1.13 sys.stdout.flush()
1.14
1.15 -THRESHOLD = 86400
1.16 -RETRYCOUNT = 3
1.17 -forcebefore = int(time.time()) - THRESHOLD
1.18 +def update_user(db, user):
1.19 + with db:
1.20 + # cannot rely on existing (possibly unsynchronized) stat.
1.21 + prevnplays = sum(sum(result.nplays) for _, result in
1.22 + db.query(sqlfunc.max(MusicResult.time), MusicResult)
1.23 + .filter(and_(MusicResult.sid == user.sid,
1.24 + MusicResult.mid == Music.mid))
1.25 + .group_by(MusicResult.mid).order_by(Music.order))
1.26 + _, newstat, _ = db.update_user(user.sid)
1.27 + nplays = newstat.nplays
1.28 + assert nplays >= prevnplays
1.29
1.30 -log('--- start of process (pid=%d) ---' % os.getpid())
1.31 -db = Database()
1.32 -userlist = db.query(User).all()
1.33 -for retrycount in xrange(1, RETRYCOUNT+1):
1.34 - retrylist = []
1.35 + prevmusiclist = [i for i, in db.query(MusicResult.mid)
1.36 + .filter(MusicResult.sid == user.sid)
1.37 + .order_by(MusicResult.time.desc())
1.38 + .group_by(MusicResult.mid)]
1.39
1.40 - for user in userlist:
1.41 - time.sleep(1)
1.42 - sid = user.sid
1.43 - log('updating user %d' % sid)
1.44 - try:
1.45 - with db:
1.46 - # cannot rely on existing (possibly unsynchronized) stat.
1.47 - prevnplays = sum(sum(result.nplays) for _, result in
1.48 - db.query(sqlfunc.max(MusicResult.time), MusicResult)
1.49 - .filter(and_(MusicResult.sid == sid,
1.50 - MusicResult.mid == Music.mid))
1.51 - .group_by(MusicResult.mid).order_by(Music.order))
1.52 - _, newstat, _ = db.update_user(sid)
1.53 - nplays = newstat.nplays
1.54 - assert nplays >= prevnplays
1.55 + rescount = db.query(MusicResult).filter(MusicResult.sid == user.sid).count()
1.56 + if rescount == 0 or nplays > prevnplays:
1.57 + if rescount == 0:
1.58 + log('.. first update scheduled')
1.59 + prevnplays = 0 # treat as initial state!
1.60 + else:
1.61 + log('.. nplays changed (%d->%d)' % (prevnplays, nplays))
1.62 + musiclist = set(db.list_avail_music(user.sid).keys())
1.63 + assert musiclist.issuperset(prevmusiclist)
1.64 + musicdiff = musiclist.difference(prevmusiclist)
1.65 + for mid in musicdiff:
1.66 + result, _ = db.update_musicresult(user.sid, mid)
1.67 + prevnplays += sum(result.nplays)
1.68 + log('.. updating new music %d (0->%d; %d plays remaining)' %
1.69 + (mid, sum(result.nplays), nplays - prevnplays))
1.70
1.71 - prevmusiclist = [i for i, in db.query(MusicResult.mid)
1.72 - .filter(MusicResult.sid == sid)
1.73 - .order_by(MusicResult.time.desc())
1.74 - .group_by(MusicResult.mid)]
1.75 + for mid in prevmusiclist: # in the order of most recent plays
1.76 + if prevnplays >= nplays:
1.77 + withinthreshold = (db.query(MusicResultRank)
1.78 + .filter(and_(MusicResultRank.sid == user.sid,
1.79 + MusicResultRank.mid == mid,
1.80 + MusicResultRank.time >= forcebefore))
1.81 + .count())
1.82 + if withinthreshold > 0: continue
1.83 + flag = True
1.84 + else:
1.85 + flag = False
1.86 + prevresult = (db.query(MusicResult)
1.87 + .filter(and_(MusicResult.sid == user.sid,
1.88 + MusicResult.mid == mid))
1.89 + .order_by(MusicResult.time.desc()).first())
1.90 + prevresnplays = 0 if prevresult is None else sum(prevresult.nplays)
1.91 + time.sleep(0.5)
1.92 + result, _ = db.update_musicresult(user.sid, mid)
1.93 + resnplays = sum(result.nplays)
1.94 + assert resnplays >= prevresnplays
1.95 + prevnplays += resnplays - prevresnplays
1.96 + log('.. updating %s %d (%d->%d; %d plays remaining)' %
1.97 + (('music', 'rank-expired music')[flag], mid, prevresnplays,
1.98 + resnplays, nplays - prevnplays))
1.99
1.100 - rescount = db.query(MusicResult).filter(MusicResult.sid == sid).count()
1.101 - if rescount == 0 or nplays > prevnplays:
1.102 - if rescount == 0:
1.103 - log('.. first update scheduled')
1.104 - prevnplays = 0 # treat as initial state!
1.105 - else:
1.106 - log('.. nplays changed (%d->%d)' % (prevnplays, nplays))
1.107 - musiclist = set(db.list_avail_music(sid).keys())
1.108 - assert musiclist.issuperset(prevmusiclist)
1.109 - musicdiff = musiclist.difference(prevmusiclist)
1.110 - for mid in musicdiff:
1.111 - result, _ = db.update_musicresult(sid, mid)
1.112 - prevnplays += sum(result.nplays)
1.113 - log('.. updating new music %d (0->%d; %d plays remaining)' %
1.114 - (mid, sum(result.nplays), nplays - prevnplays))
1.115 + if nplays != prevnplays:
1.116 + # theoretically this is an error. but in the reality, there are cases
1.117 + # that other music is played while crawling process, thus rendering
1.118 + # assertion invalid. so we just log this incident. if the situation
1.119 + # is over, nplays and prevnplays should be synchronized and this
1.120 + # warning won't be repeated.
1.121 + log('.. warning: nplays=%d and prevnplays=%d doesn\'t match.' %
1.122 + (nplays, prevnplays))
1.123
1.124 - for mid in prevmusiclist: # in the order of most recent plays
1.125 - if prevnplays >= nplays:
1.126 - withinthreshold = (db.query(MusicResultRank)
1.127 - .filter(and_(MusicResultRank.sid == sid,
1.128 - MusicResultRank.mid == mid,
1.129 - MusicResultRank.time >= forcebefore))
1.130 - .count())
1.131 - if withinthreshold > 0: continue
1.132 - flag = True
1.133 - else:
1.134 - flag = False
1.135 - prevresult = (db.query(MusicResult)
1.136 - .filter(and_(MusicResult.sid == sid,
1.137 - MusicResult.mid == mid))
1.138 - .order_by(MusicResult.time.desc()).first())
1.139 - prevresnplays = 0 if prevresult is None else sum(prevresult.nplays)
1.140 - time.sleep(0.5)
1.141 - result, _ = db.update_musicresult(sid, mid)
1.142 - resnplays = sum(result.nplays)
1.143 - assert resnplays >= prevresnplays
1.144 - prevnplays += resnplays - prevresnplays
1.145 - log('.. updating %s %d (%d->%d; %d plays remaining)' %
1.146 - (('music', 'rank-expired music')[flag], mid, prevresnplays,
1.147 - resnplays, nplays - prevnplays))
1.148 +def process():
1.149 + log('--- start of process (pid=%d) ---' % os.getpid())
1.150
1.151 - if nplays != prevnplays:
1.152 - # theoretically this is an error. but in the reality, there are cases
1.153 - # that other music is played while crawling process, thus rendering
1.154 - # assertion invalid. so we just log this incident. if the situation
1.155 - # is over, nplays and prevnplays should be synchronized and this
1.156 - # warning won't be repeated.
1.157 - log('.. warning: nplays=%d and prevnplays=%d doesn\'t match.' %
1.158 - (nplays, prevnplays))
1.159 + db = Database()
1.160 + userlist = db.query(User).all()
1.161 + for retrycount in xrange(1, RETRYCOUNT+1):
1.162 + retrylist = []
1.163
1.164 - except Exception:
1.165 - traceback.print_exc()
1.166 - log('.. crawling terminated due to exception!')
1.167 - retrylist.append(user)
1.168 + for user in userlist:
1.169 + time.sleep(1)
1.170 + log('updating user %d' % user.sid)
1.171 + try:
1.172 + update_user(db, user)
1.173 + except Exception:
1.174 + traceback.print_exc()
1.175 + log('.. crawling terminated due to exception!')
1.176 + retrylist.append(user)
1.177
1.178 - if not retrylist: break
1.179 - log('some crawler process failed due to exception: %s' %
1.180 - ', '.join(str(u.sid) for u in retrylist))
1.181 - if retrycount == RETRYCOUNT:
1.182 - log('retry limit exceeded: blockage or protocol change?')
1.183 - else:
1.184 - log('restarting crawler process soon...')
1.185 - time.sleep(10)
1.186 - userlist = retrylist
1.187 + if not retrylist: break
1.188 + log('some crawler process failed due to exception: %s' %
1.189 + ', '.join(str(u.sid) for u in retrylist))
1.190 + if retrycount == RETRYCOUNT:
1.191 + log('retry limit exceeded: blockage or protocol change?')
1.192 + else:
1.193 + log('restarting crawler process soon...')
1.194 + time.sleep(10)
1.195 + userlist = retrylist
1.196
1.197 -log('--- end of process ---')
1.198 + log('--- end of process ---')
1.199
1.200 +if __name__ == '__main__':
1.201 + process()
1.202 +
2.1 --- a/tmpl/new_result.en.html Fri Jul 24 02:37:53 2009 +0900
2.2 +++ b/tmpl/new_result.en.html Sat Jul 25 00:59:05 2009 +0900
2.3 @@ -1,6 +1,6 @@
2.4 <%inherit file="layout.en.html" />
2.5 <%def name="title()">Register</%def>
2.6 <h2>Registration success</h2>
2.7 -<p>From now on you can access <a href="${root}/u/${sid|encode_sid}">your play data</a>. (Warning: You cannot register the same friend ID again, so please bookmark the play data page in your favorites!)</p>
2.8 +<p>From now on you can access <a href="${root}/u/${sid|encode_sid}">your play data</a>. (Warning: <big>You cannot register the same friend ID again, so please make sure that the page is in your favorites!</big>)</p>
2.9
2.10 ## vim: syn=mako
3.1 --- a/tmpl/new_result.ko.html Fri Jul 24 02:37:53 2009 +0900
3.2 +++ b/tmpl/new_result.ko.html Sat Jul 25 00:59:05 2009 +0900
3.3 @@ -1,6 +1,6 @@
3.4 <%inherit file="layout.ko.html" />
3.5 <%def name="title()">등록</%def>
3.6 <h2>등록 성공</h2>
3.7 -<p>이제부터 <a href="${root}/u/${sid|encode_sid}">여러분의 플레이 기록</a>을 확인할 수 있습니다. (주의: 한 번 등록하면 같은 프렌드 아이디를 다시 등록할 수 없으므로 기록 페이지는 필히 즐겨찾기에 추가해 주세요!)</p>
3.8 +<p>이제부터 <a href="${root}/u/${sid|encode_sid}">여러분의 플레이 기록</a>을 확인할 수 있습니다. (주의: <big>한 번 등록하면 같은 프렌드 아이디를 다시 등록할 수 없으므로 기록 페이지는 필히 즐겨찾기에 추가해 주세요!</big>)</p>
3.9
3.10 ## vim: syn=mako
