From 9b20887cc29ac30bb6bcebac21cd0c618a3cadab Mon Sep 17 00:00:00 2001 From: smshine Date: Sun, 8 Feb 2026 11:23:34 +0800 Subject: [PATCH] upgrade to python3 --- judge | 8 +++- judge-daemon | 9 +++-- judge-daemonold | 67 -------------------------------- judgeold | 67 -------------------------------- python/datasource.py | 89 +++++++++++++++++++++++++++++++------------ python/entity.py | 38 +++++++++++++++--- python/judgescript.py | 8 ++-- python/tester.py | 33 ++++++++-------- utils/bitoj_adduser | 15 +++++--- 9 files changed, 139 insertions(+), 195 deletions(-) delete mode 100755 judge-daemonold delete mode 100755 judgeold diff --git a/judge b/judge index 05ffbcc..675053b 100755 --- a/judge +++ b/judge @@ -1,4 +1,5 @@ -#!/usr/bin/env python2.6 +#!/usr/bin/env python3 + import os, signal, sys, logging, logging.handlers judge = None def setup_logging(): @@ -43,10 +44,13 @@ def main(): config = getConfig() config.judgehome = judgehome vars = {'config': config} - execfile(sys.argv[1], vars, vars) + with open(sys.argv[1], 'rb') as conf_file: + code = conf_file.read() + exec(compile(code, sys.argv[1], 'exec'), vars, vars) setup_logging() + from engine import JudgeEngine judge = JudgeEngine() diff --git a/judge-daemon b/judge-daemon index c6c6d08..b32281f 100755 --- a/judge-daemon +++ b/judge-daemon @@ -1,4 +1,5 @@ -#!/usr/bin/env python2.6 +#!/usr/bin/env python3 + import os, signal, sys, time from resource import setrlimit, RLIMIT_AS @@ -38,7 +39,8 @@ def main(): pid = configs[conf] ret = os.waitpid(pid, os.WNOHANG) if ret[0]: - print >>sys.stderr, 'judge process %d dead, restarting...' % pid + print('judge process %d dead, restarting...' % pid, file=sys.stderr) + configs[conf] = run(conf) time.sleep(1) @@ -47,7 +49,8 @@ def run(conf): if pid: return pid else: - cmd = '/usr/bin/python' + cmd = sys.executable + conf = os.path.join('conf', conf) os.execvp(cmd, (cmd, 'judge', conf)) diff --git a/judge-daemonold b/judge-daemonold deleted file mode 100755 index fba9b95..0000000 --- a/judge-daemonold +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python2.6 - -import os, signal, sys, time -from resource import setrlimit, RLIMIT_AS - -configs = {} -quitnow = False -judgehome = os.path.abspath(os.path.dirname(sys.argv[0])) - -def main(): - os.chdir(judgehome) - - # redirect stdout and stderr to file - os.close(1) - fd = os.open('log/daemon.log', os.O_WRONLY | os.O_APPEND | os.O_CREAT) - os.close(2) - fd = os.dup(fd) - - # setup resource usage - setrlimit(RLIMIT_AS, (1024 * 1024 * 256, 1024 * 1024 * 256)) - - # setup signal handler - signal.signal(signal.SIGINT, quit) - signal.signal(signal.SIGQUIT, quit) - - # find config files - files = os.listdir(os.path.join(judgehome, 'conf')) - for filename in files: - if filename[:5] == 'conf-' and filename[-3:] == '.py': - configs[filename] = 0 - - # start all the judge process - for conf in configs.keys(): - configs[conf] = run(conf) - - # check for the dead of judge process - while not quitnow: - for conf in configs.keys(): - pid = configs[conf] - ret = os.waitpid(pid, os.WNOHANG) - if ret[0]: - print >>sys.stderr, 'judge process %d dead, restarting...' % pid - configs[conf] = run(conf) - time.sleep(1) - -def run(conf): - pid = os.fork() - if pid: - return pid - else: - cmd = '/usr/bin/python2.6' - conf = os.path.join('conf', conf) - os.execvp(cmd, (cmd, 'judge', conf)) - -def quit(sig, frame): - quitnow = True - for conf in configs.keys(): - pid = configs[conf] - os.kill(pid, sig) - - for conf in configs.keys(): - pid = configs[conf] - os.waitpid(pid, 0) - exit() - -if __name__ == '__main__': - main() diff --git a/judgeold b/judgeold deleted file mode 100755 index d4be3ab..0000000 --- a/judgeold +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python2.6 - -import os, signal, sys, logging, logging.handlers - -judge = None - -def setup_logging(): - from engineconfig import getConfig - config = getConfig() - - judgelog = os.path.join(config.judgehome, config.logdir, - 'judge-%s.log' % config.profile) - testerlog = os.path.join(config.judgehome, config.logdir, - 'tester-%s.log' % config.profile) - - # setup logger - # judge.main is for main judge program - logger = logging.getLogger('main') - hdlr = logging.handlers.RotatingFileHandler( - judgelog, 'a', 1024 * 1024 * 5, 4) - formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') - hdlr.setFormatter(formatter) - logger.addHandler(hdlr) - if config.debug: - logger.setLevel(logging.DEBUG) - else: - logger.setLevel(logging.INFO) - - # judge.tester is for forked compiler and programs - logger = logging.getLogger('tester') - hdlr = logging.handlers.RotatingFileHandler( - testerlog, 'a', 1024 * 1024 * 5, 4) - formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') - hdlr.setFormatter(formatter) - logger.addHandler(hdlr) - if config.debug: - logger.setLevel(logging.DEBUG) - else: - logger.setLevel(logging.INFO) - -def main(): - judgehome = os.path.abspath(os.path.dirname(sys.argv[0])) - sys.path.insert(1, os.path.join(judgehome, 'python')) - - from engineconfig import getConfig - config = getConfig() - config.judgehome = judgehome - vars = {'config': config} - - execfile(sys.argv[1], vars, vars) - - setup_logging() - - from engine import JudgeEngine - judge = JudgeEngine() - - # setup signal handler - signal.signal(signal.SIGINT, judge.quit) - signal.signal(signal.SIGQUIT, judge.quit) - signal.signal(signal.SIGTERM, judge.quit) - - judge.run() - -if __name__ == '__main__': - main() - -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/python/datasource.py b/python/datasource.py index 2c84073..4048e36 100755 --- a/python/datasource.py +++ b/python/datasource.py @@ -14,6 +14,31 @@ from engineconfig import getConfig from entity import Submit, Problem, TestCase, PresetCode, DataFile +def _to_bytes(value, encoding='utf-8', errors='replace'): + if value is None: + return b'' + if isinstance(value, xmlrpclib.Binary): + return value.data + if isinstance(value, bytes): + return value + if isinstance(value, str): + return value.encode(encoding, errors) + return str(value).encode(encoding, errors) + + +def _to_text(value, encoding='utf-8', errors='replace'): + if value is None: + return '' + if isinstance(value, xmlrpclib.Binary): + value = value.data + if isinstance(value, bytes): + return value.decode(encoding, errors) + if isinstance(value, str): + return value + return str(value) + + + class DataSourceError(Exception): pass @@ -244,9 +269,10 @@ class XmlRpcDataSource: try: submits = self.server.oj.get_submits(judgeid, limit) for submit in submits: - if not isinstance(submit['code'], str): - submit['code'] = submit['code'].__str__() + if not isinstance(submit.get('code'), str): + submit['code'] = _to_text(submit.get('code')) return submits + except xmlrpclib.Error as e: raise DataSourceError(e) except socket.error as e: @@ -268,12 +294,13 @@ class XmlRpcDataSource: try: tests = self.server.oj.get_tests(problemid, full) for test in tests: - if not isinstance(test['input'], str): - test['input'] = test['input'].__str__() - if not isinstance(test['output'], str): - test['output'] = test['output'].__str__() + if not isinstance(test.get('input'), str): + test['input'] = _to_text(test.get('input')) + if not isinstance(test.get('output'), str): + test['output'] = _to_text(test.get('output')) self.logger.debug('Got %d test case(s)', len(tests)) return tests + except xmlrpclib.Error as e: raise DataSourceError(e) except socket.error as e: @@ -284,13 +311,16 @@ class XmlRpcDataSource: while True: try: test = self.server.oj.get_gztest(testid) - if not isinstance(test['input'], str): - test['input'] = test['input'].__str__() - if not isinstance(test['output'], str): - test['output'] = test['output'].__str__() - test['input'] = bz2.decompress(test['input']) - test['output'] = bz2.decompress(test['output']) + try: + input_data = bz2.decompress(_to_bytes(test.get('input'))) + output_data = bz2.decompress(_to_bytes(test.get('output'))) + except OSError: + self.logger.exception('Failed to decompress test data for %s', testid) + raise DataSourceError('Invalid bz2 data for test %s' % testid) + test['input'] = _to_text(input_data) + test['output'] = _to_text(output_data) return test + except xmlrpclib.Error as e: raise DataSourceError(e) except socket.error as e: @@ -302,10 +332,11 @@ class XmlRpcDataSource: try: codes = self.server.oj.get_presetcodes(problemid, lang) for code in codes: - if not isinstance(code['code'], str): - code['code'] = code['code'].__str__() + if not isinstance(code.get('code'), str): + code['code'] = _to_text(code.get('code')) self.logger.debug('Got %d presetcodes', len(codes)) return codes + except xmlrpclib.Error as e: raise DataSourceError(e) except socket.error as e: @@ -328,7 +359,8 @@ class XmlRpcDataSource: while True: try: data = self.server.oj.get_datafile_data(datafileid) - return str(data) + return _to_bytes(data) + except xmlrpclib.Error as e: raise DataSourceError(e) except socket.error as e: @@ -336,7 +368,8 @@ class XmlRpcDataSource: time.sleep(self.config.retry_wait) def update_submit_compilemessage(self, id, compilemsg): - compilemsg = xmlrpclib.Binary(compilemsg) + compilemsg = xmlrpclib.Binary(_to_bytes(compilemsg)) + while True: try: return self.server.oj.update_submit_compilemessage( @@ -349,10 +382,11 @@ class XmlRpcDataSource: def update_submit_test_results(self, id, results): for r in results: - if not isinstance(r['stdout'], str): r['stdout'] = '' - if not isinstance(r['stderr'], str): r['stderr'] = '' - r['stdout'] = xmlrpclib.Binary(r['stdout']) - r['stderr'] = xmlrpclib.Binary(r['stderr']) + stdout = r.get('stdout', '') + stderr = r.get('stderr', '') + r['stdout'] = xmlrpclib.Binary(_to_bytes(stdout)) + r['stderr'] = xmlrpclib.Binary(_to_bytes(stderr)) + while True: try: return self.server.oj.update_submit_test_results(id, results) @@ -389,10 +423,8 @@ class XmlRpcDataSource: while True: try: msg = self.server.oj.get_submit_compilemessage(sid) - if isinstance(msg, str): - return msg - else: - return msg.__str__() + return _to_text(msg) + except xmlrpclib.Error as e: return DataSourceError(e) except socket.error as e: @@ -411,7 +443,16 @@ class XmlRpcDataSource: class DataSourceTest(unittest.TestCase): + def testByteTextConversion(self): + raw = b'hello' + self.assertEqual(_to_text(raw), 'hello') + self.assertEqual(_to_bytes('hello'), b'hello') + binval = xmlrpclib.Binary(b'world') + self.assertEqual(_to_text(binval), 'world') + self.assertEqual(_to_bytes(binval), b'world') + def setUp(self): + exec(open(os.path.join('..', 'testdata', 'test_config.py')).read()) self.config = getConfig() self.datasource = self.config.datasources[0] diff --git a/python/entity.py b/python/entity.py index c3ad174..d3c5684 100755 --- a/python/entity.py +++ b/python/entity.py @@ -5,6 +5,17 @@ import unittest from engineconfig import getConfig from judgescript import InternalJudge, ExternalJudge + +def _to_text(value, encoding='utf-8', errors='replace'): + if value is None: + return '' + if isinstance(value, bytes): + return value.decode(encoding, errors) + if isinstance(value, str): + return value + return str(value) + + class Problem: def __init__(self, datasource, row): @@ -17,12 +28,13 @@ class Problem: self.timemodified = row['timemodified'] self.vcode = row['validator_code'] if not isinstance(self.vcode, str): - self.vcode = self.vcode.__str__() + self.vcode = _to_text(self.vcode) self.vtype = row['validator_type'] self.vlang = row['validator_lang'] self.gcode = row['generator_code'] if not isinstance(self.gcode, str): - self.gcode = self.vcode.__str__() + self.gcode = _to_text(self.gcode) + self.gtype = row['generator_type'] self.standard_code = row['standard_code'] @@ -187,8 +199,9 @@ class TestCase: if inmtime <= self.timemodified or outmtime <= self.timemodified: logger.debug('Creating input/output file %s and %s' % (self.infile, self.outfile)) row = datasource.get_test(self.id) - input = string.replace(row['input'], '\r\n', '\n') - output = string.replace(row['output'], '\r\n', '\n') + input = row['input'].replace('\r\n', '\n') + output = row['output'].replace('\r\n', '\n') + with open(self.infile, 'w') as f: f.write(input) @@ -239,6 +252,7 @@ class DataFile: # Save datafile config = getConfig() + logger = logging.getLogger('main') testdir = os.path.join(config.datadir, 'testcase') if not os.path.exists(testdir): os.mkdir(testdir) self.absolute_path = os.path.join( @@ -251,16 +265,28 @@ class DataFile: mtime = os.stat(self.absolute_path)[stat.ST_MTIME] if mtime < self.timemodified: data = datasource.get_datafile_data(self.id) - data = bz2.decompress(data) + try: + data = bz2.decompress(data) + except OSError: + logger.exception('Failed to decompress datafile %s', self.id) + DataFile.write_lock.release() + raise if self.type == 'text': + if isinstance(data, (bytes, bytearray)): + text = data.decode('utf-8', errors='replace') + else: + text = str(data) with open(self.absolute_path, 'w') as f: - f.write(string.replace(data, '\r\n', '\n')) + f.write(text.replace('\r\n', '\n')) else: + if isinstance(data, str): + data = data.encode('utf-8', errors='replace') with open(self.absolute_path, 'wb') as f: f.write(data) DataFile.write_lock.release() + if __name__ == '__main__': unittest.main() diff --git a/python/judgescript.py b/python/judgescript.py index bde37e8..d06d233 100755 --- a/python/judgescript.py +++ b/python/judgescript.py @@ -80,10 +80,11 @@ class ExternalJudge: self.comparecmd = tester.comparecmd self.codefile = os.path.abspath(os.path.join(datadir, tester.source)) with open(self.codefile, 'w') as f: - f.write(string.replace(vcode, '\r\n', '\n')) + f.write(vcode.replace('\r\n', '\n')) if len(vcode) > 0 and vcode[-1] != '\n': f.write('\n') + self.logger.debug("Save validator code as %s" % self.codefile) def judge(self, sid, tid, tin, tout, result, errfile, rundir = None): @@ -116,9 +117,10 @@ class ExternalJudge: pid = os.fork() if pid == 0: os.close(1) - os.open(rfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0666) + os.open(rfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666) os.close(2) - os.open(errfile, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0666) + os.open(errfile, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o666) + if rundir: os.chdir(rundir) os.execv(cmd[0], cmd) diff --git a/python/tester.py b/python/tester.py index 1912a5f..628bae7 100755 --- a/python/tester.py +++ b/python/tester.py @@ -298,8 +298,9 @@ class SimpleTester(TesterBase): shutil.copyfile(os.path.join(rundir, submit_output_filename), outfile) except IOError: - f = file(outfile, 'w') - f.close() + with open(outfile, 'w') as f: + f.write('') + ret = [testcase.id, exitcode, sig, outfile, errfile, timeused, memused] if timeused > testcase.timelimit: @@ -484,21 +485,19 @@ class SimpleTesterTestCase(OJTestCase): self.assertEqual(r[1], 'AC') self.assertEqual(r[2], 0) self.assertEqual(r[3], 0) - f = file(r[4], 'r') - o = string.join(f.readlines(), '\n') - f.close() + with open(r[4], 'rb') as f: + o = f.read().decode('latin1') self.assertEqual(o, '3\n') - f = file(r[5], 'r') - o = string.join(f.readlines(), '\n') - f.close() + with open(r[5], 'rb') as f: + o = f.read().decode('latin1') self.assertEqual(o, '') r = self.st.run(submit, testcases[1]) - f = file(r[4], 'r') - o = string.join(f.readlines(), '\n') - f.close() + with open(r[4], 'rb') as f: + o = f.read().decode('latin1') self.assertEqual(o, '4\n') + self.st.cleanup(submit) self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir)) @@ -621,11 +620,11 @@ class SimpleTesterTestCase(OJTestCase): self.assertTrue(r) testcases = submit.get_testcases() r = self.st.run(submit, testcases[0]) - f = file(os.path.join(datadir, '0000000001.out')) - o = string.join(f.readlines(), '\n') - f.close() + with open(os.path.join(datadir, '0000000001.out'), 'rb') as f: + o = f.read().decode('latin1') self.assertEqual(o, '3\0\n') + self.st.cleanup(submit) self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir)) @@ -640,11 +639,11 @@ class SimpleTesterTestCase(OJTestCase): self.assertTrue(r) testcases = submit.get_testcases() r = self.st.run(submit, testcases[0]) - f = file(os.path.join(datadir, '0000000001.out')) - o = string.join(f.readlines(), '\n') - f.close() + with open(os.path.join(datadir, '0000000001.out'), 'rb') as f: + o = f.read().decode('latin1') self.assertEqual(o, '\xbb') + self.st.cleanup(submit) self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir)) diff --git a/utils/bitoj_adduser b/utils/bitoj_adduser index 9c83fb7..fc88a53 100755 --- a/utils/bitoj_adduser +++ b/utils/bitoj_adduser @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ @@ -7,14 +7,16 @@ """ -import os, sys, string +import os, sys + rundir = '/var/lib/bitoj' def usage(message = None): if message: - print message - print "Usage: bitoj_adduser minnum maxnum" + print(message) + print("Usage: bitoj_adduser minnum maxnum") + def adduser(usernum): global rundir @@ -34,8 +36,9 @@ def main(): usage() sys.exit(1) - minnum = string.atoi(sys.argv[1]) - maxnum = string.atoi(sys.argv[2]) + minnum = int(sys.argv[1]) + maxnum = int(sys.argv[2]) + if minnum > maxnum: usage("minnum should be small than maxnum")