upgrade to python3

judge

@@ -1,4 +1,5 @@
-#!/usr/bin/env python2.6
+#!/usr/bin/env python3
+
 import os, signal, sys, logging, logging.handlers
 judge = None
 def setup_logging():
@@ -43,10 +44,13 @@ def main():
     config = getConfig()
     config.judgehome = judgehome
     vars = {'config': config}
-    execfile(sys.argv[1], vars, vars)
+    with open(sys.argv[1], 'rb') as conf_file:
+        code = conf_file.read()
+    exec(compile(code, sys.argv[1], 'exec'), vars, vars)
 
     setup_logging()
 
+
     from engine import JudgeEngine
     judge = JudgeEngine()
 

judge-daemon

@@ -1,4 +1,5 @@
-#!/usr/bin/env python2.6
+#!/usr/bin/env python3
+
 
 import os, signal, sys, time
 from resource import setrlimit, RLIMIT_AS
@@ -38,7 +39,8 @@ def main():
             pid = configs[conf]
             ret = os.waitpid(pid, os.WNOHANG)
             if ret[0]:
-                print >>sys.stderr, 'judge process %d dead, restarting...' % pid
+                print('judge process %d dead, restarting...' % pid, file=sys.stderr)
+
                 configs[conf] = run(conf)
         time.sleep(1)
 
@@ -47,7 +49,8 @@ def run(conf):
     if pid:
         return pid
     else:
-        cmd = '/usr/bin/python'
+        cmd = sys.executable
+
         conf = os.path.join('conf', conf)
         os.execvp(cmd, (cmd, 'judge', conf))
 

judge-daemonold

@@ -1,67 +0,0 @@
-#!/usr/bin/env python2.6
-
-import os, signal, sys, time
-from resource import setrlimit, RLIMIT_AS
-
-configs = {}
-quitnow = False
-judgehome = os.path.abspath(os.path.dirname(sys.argv[0]))
-
-def main():
-    os.chdir(judgehome)
-
-    # redirect stdout and stderr to file
-    os.close(1)
-    fd = os.open('log/daemon.log', os.O_WRONLY | os.O_APPEND | os.O_CREAT)
-    os.close(2)
-    fd = os.dup(fd)
-
-    # setup resource usage
-    setrlimit(RLIMIT_AS, (1024 * 1024 * 256, 1024 * 1024 * 256))
-
-    # setup signal handler
-    signal.signal(signal.SIGINT, quit)
-    signal.signal(signal.SIGQUIT, quit)
-
-    # find config files
-    files = os.listdir(os.path.join(judgehome, 'conf'))
-    for filename in files:
-        if filename[:5] == 'conf-' and filename[-3:] == '.py':
-            configs[filename] = 0
-
-    # start all the judge process
-    for conf in configs.keys():
-        configs[conf] = run(conf)
-
-    # check for the dead of judge process
-    while not quitnow:
-        for conf in configs.keys():
-            pid = configs[conf]
-            ret = os.waitpid(pid, os.WNOHANG)
-            if ret[0]:
-                print >>sys.stderr, 'judge process %d dead, restarting...' % pid
-                configs[conf] = run(conf)
-        time.sleep(1)
-
-def run(conf):
-    pid = os.fork()
-    if pid:
-        return pid
-    else:
-        cmd = '/usr/bin/python2.6'
-        conf = os.path.join('conf', conf)
-        os.execvp(cmd, (cmd, 'judge', conf))
-
-def quit(sig, frame):
-    quitnow = True
-    for conf in configs.keys():
-        pid = configs[conf]
-        os.kill(pid, sig)
-
-    for conf in configs.keys():
-        pid = configs[conf]
-        os.waitpid(pid, 0)
-    exit()
-
-if __name__ == '__main__':
-    main()

judgeold

@@ -1,67 +0,0 @@
-#!/usr/bin/env python2.6
-
-import os, signal, sys, logging, logging.handlers
-
-judge = None
-
-def setup_logging():
-    from engineconfig import getConfig
-    config = getConfig()
-
-    judgelog = os.path.join(config.judgehome, config.logdir,
-                            'judge-%s.log' % config.profile)
-    testerlog = os.path.join(config.judgehome, config.logdir,
-                             'tester-%s.log' % config.profile)
-
-    # setup logger
-    # judge.main is for main judge program
-    logger = logging.getLogger('main')
-    hdlr = logging.handlers.RotatingFileHandler(
-        judgelog, 'a', 1024 * 1024 * 5, 4)
-    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
-    hdlr.setFormatter(formatter)
-    logger.addHandler(hdlr)
-    if config.debug:
-        logger.setLevel(logging.DEBUG)
-    else:
-        logger.setLevel(logging.INFO)
-    
-    # judge.tester is for forked compiler and programs
-    logger = logging.getLogger('tester')
-    hdlr = logging.handlers.RotatingFileHandler(
-        testerlog, 'a', 1024 * 1024 * 5, 4)
-    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
-    hdlr.setFormatter(formatter)
-    logger.addHandler(hdlr)
-    if config.debug:
-        logger.setLevel(logging.DEBUG)
-    else:
-        logger.setLevel(logging.INFO)
-
-def main():
-    judgehome = os.path.abspath(os.path.dirname(sys.argv[0]))
-    sys.path.insert(1, os.path.join(judgehome, 'python'))
-
-    from engineconfig import getConfig
-    config = getConfig()
-    config.judgehome = judgehome
-    vars = {'config': config}
-
-    execfile(sys.argv[1], vars, vars)
-
-    setup_logging()
-
-    from engine import JudgeEngine
-    judge = JudgeEngine()
-
-    # setup signal handler
-    signal.signal(signal.SIGINT, judge.quit)
-    signal.signal(signal.SIGQUIT, judge.quit)
-    signal.signal(signal.SIGTERM, judge.quit)
-
-    judge.run()
-    
-if __name__ == '__main__':
-    main()
-
-# vim: set expandtab tabstop=4 shiftwidth=4:

python/datasource.py

@@ -14,6 +14,31 @@ from engineconfig import getConfig
 from entity import Submit, Problem, TestCase, PresetCode, DataFile
 
 
+def _to_bytes(value, encoding='utf-8', errors='replace'):
+    if value is None:
+        return b''
+    if isinstance(value, xmlrpclib.Binary):
+        return value.data
+    if isinstance(value, bytes):
+        return value
+    if isinstance(value, str):
+        return value.encode(encoding, errors)
+    return str(value).encode(encoding, errors)
+
+
+def _to_text(value, encoding='utf-8', errors='replace'):
+    if value is None:
+        return ''
+    if isinstance(value, xmlrpclib.Binary):
+        value = value.data
+    if isinstance(value, bytes):
+        return value.decode(encoding, errors)
+    if isinstance(value, str):
+        return value
+    return str(value)
+
+
+
 class DataSourceError(Exception):
     pass
 
@@ -244,9 +269,10 @@ class XmlRpcDataSource:
             try:
                 submits = self.server.oj.get_submits(judgeid, limit)
                 for submit in submits:
-                    if not isinstance(submit['code'], str):
+                    if not isinstance(submit.get('code'), str):
-                        submit['code'] = submit['code'].__str__()
+                        submit['code'] = _to_text(submit.get('code'))
                 return submits
+
             except xmlrpclib.Error as e:
                 raise DataSourceError(e)
             except socket.error as e:
@@ -268,12 +294,13 @@ class XmlRpcDataSource:
             try:
                 tests = self.server.oj.get_tests(problemid, full)
                 for test in tests:
-                    if not isinstance(test['input'], str):
+                    if not isinstance(test.get('input'), str):
-                        test['input'] = test['input'].__str__()
+                        test['input'] = _to_text(test.get('input'))
-                    if not isinstance(test['output'], str):
+                    if not isinstance(test.get('output'), str):
-                        test['output'] = test['output'].__str__()
+                        test['output'] = _to_text(test.get('output'))
                 self.logger.debug('Got %d test case(s)', len(tests))
                 return tests
+
             except xmlrpclib.Error as e:
                 raise DataSourceError(e)
             except socket.error as e:
@@ -284,13 +311,16 @@ class XmlRpcDataSource:
         while True:
             try:
                 test = self.server.oj.get_gztest(testid)
-                if not isinstance(test['input'], str):
+                try:
-                    test['input'] = test['input'].__str__()
+                    input_data = bz2.decompress(_to_bytes(test.get('input')))
-                if not isinstance(test['output'], str):
+                    output_data = bz2.decompress(_to_bytes(test.get('output')))
-                    test['output'] = test['output'].__str__()
+                except OSError:
-                test['input'] = bz2.decompress(test['input'])
+                    self.logger.exception('Failed to decompress test data for %s', testid)
-                test['output'] = bz2.decompress(test['output'])
+                    raise DataSourceError('Invalid bz2 data for test %s' % testid)
+                test['input'] = _to_text(input_data)
+                test['output'] = _to_text(output_data)
                 return test
+
             except xmlrpclib.Error as e:
                 raise DataSourceError(e)
             except socket.error as e:
@@ -302,10 +332,11 @@ class XmlRpcDataSource:
             try:
                 codes = self.server.oj.get_presetcodes(problemid, lang)
                 for code in codes:
-                    if not isinstance(code['code'], str):
+                    if not isinstance(code.get('code'), str):
-                        code['code'] = code['code'].__str__()
+                        code['code'] = _to_text(code.get('code'))
                 self.logger.debug('Got %d presetcodes', len(codes))
                 return codes
+
             except xmlrpclib.Error as e:
                 raise DataSourceError(e)
             except socket.error as e:
@@ -328,7 +359,8 @@ class XmlRpcDataSource:
         while True:
             try:
                 data = self.server.oj.get_datafile_data(datafileid)
-                return str(data)
+                return _to_bytes(data)
+
             except xmlrpclib.Error as e:
                 raise DataSourceError(e)
             except socket.error as e:
@@ -336,7 +368,8 @@ class XmlRpcDataSource:
                 time.sleep(self.config.retry_wait)
 
     def update_submit_compilemessage(self, id, compilemsg):
-        compilemsg = xmlrpclib.Binary(compilemsg)
+        compilemsg = xmlrpclib.Binary(_to_bytes(compilemsg))
+
         while True:
             try:
                 return self.server.oj.update_submit_compilemessage(
@@ -349,10 +382,11 @@ class XmlRpcDataSource:
 
     def update_submit_test_results(self, id, results):
         for r in results:
-            if not isinstance(r['stdout'], str): r['stdout'] = ''
+            stdout = r.get('stdout', '')
-            if not isinstance(r['stderr'], str): r['stderr'] = ''
+            stderr = r.get('stderr', '')
-            r['stdout'] = xmlrpclib.Binary(r['stdout'])
+            r['stdout'] = xmlrpclib.Binary(_to_bytes(stdout))
-            r['stderr'] = xmlrpclib.Binary(r['stderr'])
+            r['stderr'] = xmlrpclib.Binary(_to_bytes(stderr))
+
         while True:
             try:
                 return self.server.oj.update_submit_test_results(id, results)
@@ -389,10 +423,8 @@ class XmlRpcDataSource:
         while True:
             try:
                 msg = self.server.oj.get_submit_compilemessage(sid)
-                if isinstance(msg, str):
+                return _to_text(msg)
-                    return msg
+
-                else:
-                    return msg.__str__()
             except xmlrpclib.Error as e:
                 return DataSourceError(e)
             except socket.error as e:
@@ -411,7 +443,16 @@ class XmlRpcDataSource:
 
 class DataSourceTest(unittest.TestCase):
 
+    def testByteTextConversion(self):
+        raw = b'hello'
+        self.assertEqual(_to_text(raw), 'hello')
+        self.assertEqual(_to_bytes('hello'), b'hello')
+        binval = xmlrpclib.Binary(b'world')
+        self.assertEqual(_to_text(binval), 'world')
+        self.assertEqual(_to_bytes(binval), b'world')
+
     def setUp(self):
+
         exec(open(os.path.join('..', 'testdata', 'test_config.py')).read())
         self.config = getConfig()
         self.datasource = self.config.datasources[0]

python/entity.py

@@ -5,6 +5,17 @@ import unittest
 from engineconfig import getConfig
 from judgescript import InternalJudge, ExternalJudge
 
+
+def _to_text(value, encoding='utf-8', errors='replace'):
+    if value is None:
+        return ''
+    if isinstance(value, bytes):
+        return value.decode(encoding, errors)
+    if isinstance(value, str):
+        return value
+    return str(value)
+
+
 class Problem:
     
     def __init__(self, datasource, row):
@@ -17,12 +28,13 @@ class Problem:
         self.timemodified = row['timemodified']
         self.vcode = row['validator_code']
         if not isinstance(self.vcode, str):
-            self.vcode = self.vcode.__str__()
+            self.vcode = _to_text(self.vcode)
         self.vtype = row['validator_type']
         self.vlang = row['validator_lang']
         self.gcode = row['generator_code']
         if not isinstance(self.gcode, str):
-            self.gcode = self.vcode.__str__()
+            self.gcode = _to_text(self.gcode)
+
         self.gtype = row['generator_type']
         self.standard_code = row['standard_code']
 
@@ -187,8 +199,9 @@ class TestCase:
         if inmtime <= self.timemodified or outmtime <= self.timemodified:
             logger.debug('Creating input/output file %s and %s' % (self.infile, self.outfile))
             row = datasource.get_test(self.id)
-            input = string.replace(row['input'], '\r\n', '\n')
+            input = row['input'].replace('\r\n', '\n')
-            output = string.replace(row['output'], '\r\n', '\n')
+            output = row['output'].replace('\r\n', '\n')
+
 
             with open(self.infile, 'w') as f:
                 f.write(input)
@@ -239,6 +252,7 @@ class DataFile:
 
         # Save datafile
         config = getConfig()
+        logger = logging.getLogger('main')
         testdir = os.path.join(config.datadir, 'testcase')
         if not os.path.exists(testdir): os.mkdir(testdir)
         self.absolute_path = os.path.join(
@@ -251,16 +265,28 @@ class DataFile:
             mtime = os.stat(self.absolute_path)[stat.ST_MTIME] 
         if mtime < self.timemodified:
             data = datasource.get_datafile_data(self.id)
-            data = bz2.decompress(data)
+            try:
+                data = bz2.decompress(data)
+            except OSError:
+                logger.exception('Failed to decompress datafile %s', self.id)
+                DataFile.write_lock.release()
+                raise
             if self.type == 'text':
+                if isinstance(data, (bytes, bytearray)):
+                    text = data.decode('utf-8', errors='replace')
+                else:
+                    text = str(data)
                 with open(self.absolute_path, 'w') as f:
-                    f.write(string.replace(data, '\r\n', '\n'))
+                    f.write(text.replace('\r\n', '\n'))
             else:
+                if isinstance(data, str):
+                    data = data.encode('utf-8', errors='replace')
                 with open(self.absolute_path, 'wb') as f:
                     f.write(data)
 
         DataFile.write_lock.release()
 
+
 if __name__ == '__main__':
     unittest.main()
 

python/judgescript.py

@@ -80,10 +80,11 @@ class ExternalJudge:
         self.comparecmd = tester.comparecmd
         self.codefile = os.path.abspath(os.path.join(datadir, tester.source))
         with open(self.codefile, 'w') as f:
-            f.write(string.replace(vcode, '\r\n', '\n'))
+            f.write(vcode.replace('\r\n', '\n'))
             if len(vcode) > 0 and vcode[-1] != '\n':
                 f.write('\n')
 
+
         self.logger.debug("Save validator code as %s" % self.codefile)
 
     def judge(self, sid, tid, tin, tout, result, errfile, rundir = None):
@@ -116,9 +117,10 @@ class ExternalJudge:
         pid = os.fork()
         if pid == 0:
             os.close(1)
-            os.open(rfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0666)
+            os.open(rfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666)
             os.close(2)
-            os.open(errfile, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0666)
+            os.open(errfile, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o666)
+
 
             if rundir: os.chdir(rundir)
             os.execv(cmd[0], cmd)

python/tester.py

@@ -298,8 +298,9 @@ class SimpleTester(TesterBase):
                 shutil.copyfile(os.path.join(rundir, submit_output_filename),
                                 outfile)
             except IOError:
-                f = file(outfile, 'w')
+                with open(outfile, 'w') as f:
-                f.close()
+                    f.write('')
+
 
         ret = [testcase.id, exitcode, sig, outfile, errfile, timeused, memused]
         if timeused > testcase.timelimit:
@@ -484,21 +485,19 @@ class SimpleTesterTestCase(OJTestCase):
         self.assertEqual(r[1], 'AC')
         self.assertEqual(r[2], 0)
         self.assertEqual(r[3], 0)
-        f = file(r[4], 'r')
+        with open(r[4], 'rb') as f:
-        o = string.join(f.readlines(), '\n')
+            o = f.read().decode('latin1')
-        f.close()
         self.assertEqual(o, '3\n')
-        f = file(r[5], 'r')
+        with open(r[5], 'rb') as f:
-        o = string.join(f.readlines(), '\n')
+            o = f.read().decode('latin1')
-        f.close()
         self.assertEqual(o, '')
 
         r = self.st.run(submit, testcases[1])
-        f = file(r[4], 'r')
+        with open(r[4], 'rb') as f:
-        o = string.join(f.readlines(), '\n')
+            o = f.read().decode('latin1')
-        f.close()
         self.assertEqual(o, '4\n')
 
+
         self.st.cleanup(submit)
         self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir))
 
@@ -621,11 +620,11 @@ class SimpleTesterTestCase(OJTestCase):
         self.assertTrue(r)
         testcases = submit.get_testcases()
         r = self.st.run(submit, testcases[0])
-        f = file(os.path.join(datadir, '0000000001.out'))
+        with open(os.path.join(datadir, '0000000001.out'), 'rb') as f:
-        o = string.join(f.readlines(), '\n')
+            o = f.read().decode('latin1')
-        f.close()
         self.assertEqual(o, '3\0\n')
 
+
         self.st.cleanup(submit)
         self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir))
 
@@ -640,11 +639,11 @@ class SimpleTesterTestCase(OJTestCase):
         self.assertTrue(r)
         testcases = submit.get_testcases()
         r = self.st.run(submit, testcases[0])
-        f = file(os.path.join(datadir, '0000000001.out'))
+        with open(os.path.join(datadir, '0000000001.out'), 'rb') as f:
-        o = string.join(f.readlines(), '\n')
+            o = f.read().decode('latin1')
-        f.close()
         self.assertEqual(o, '\xbb')
 
+
         self.st.cleanup(submit)
         self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir))
 

utils/bitoj_adduser

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 """
@@ -7,14 +7,16 @@
 
 """
 
-import os, sys, string
+import os, sys
+
 
 rundir = '/var/lib/bitoj'
 
 def usage(message = None):
     if message:
-        print message
+        print(message)
-    print "Usage: bitoj_adduser minnum maxnum"
+    print("Usage: bitoj_adduser minnum maxnum")
+
 
 def adduser(usernum):
     global rundir
@@ -34,8 +36,9 @@ def main():
         usage()
         sys.exit(1)
 
-    minnum = string.atoi(sys.argv[1])
+    minnum = int(sys.argv[1])
-    maxnum = string.atoi(sys.argv[2])
+    maxnum = int(sys.argv[2])
+
 
     if minnum > maxnum:
         usage("minnum should be small than maxnum")