commit e8774085f83a3d6a3f4c95726351ec97727247bc Author: Shuming Liu Date: Sat Oct 11 11:23:58 2025 +0800 first commit diff --git a/conf-demo.py b/conf-demo.py new file mode 100755 index 0000000..36ecdcf --- /dev/null +++ b/conf-demo.py @@ -0,0 +1,7 @@ +#Add XML-RPC datasource +#config.add_xmlrpc_datasource('http://11.9.36.12/mod/programming/ojfeeder.php') +config.add_xmlrpc_datasource('http://d1kt.cn/course/mod/programming/ojfeeder.php') + +# Options for debug +#config.debug = True +config.no_cleanup = True diff --git a/judge b/judge new file mode 100755 index 0000000..05ffbcc --- /dev/null +++ b/judge @@ -0,0 +1,63 @@ +#!/usr/bin/env python2.6 +import os, signal, sys, logging, logging.handlers +judge = None +def setup_logging(): + from engineconfig import getConfig + config = getConfig() + + judgelog = os.path.join(config.judgehome, config.logdir, + 'judge-%s.log' % config.profile) + testerlog = os.path.join(config.judgehome, config.logdir, + 'tester-%s.log' % config.profile) + + # setup logger + # judge.main is for main judge program + logger = logging.getLogger('main') + hdlr = logging.handlers.RotatingFileHandler( + judgelog, 'a', 1024 * 1024 * 5, 4) + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + if config.debug: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + # judge.tester is for forked compiler and programs + logger = logging.getLogger('tester') + hdlr = logging.handlers.RotatingFileHandler( + testerlog, 'a', 1024 * 1024 * 5, 4) + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + if config.debug: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + +def main(): + judgehome = os.path.abspath(os.path.dirname(sys.argv[0])) + sys.path.insert(1, os.path.join(judgehome, 'python')) + + from engineconfig import getConfig + config = getConfig() + config.judgehome = judgehome + vars = {'config': config} + execfile(sys.argv[1], vars, vars) + + setup_logging() + + from engine import JudgeEngine + judge = JudgeEngine() + + # setup signal handler + signal.signal(signal.SIGINT, judge.quit) + signal.signal(signal.SIGQUIT, judge.quit) + signal.signal(signal.SIGTERM, judge.quit) + + judge.run() + +if __name__ == '__main__': + main() + +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/judge-daemon b/judge-daemon new file mode 100755 index 0000000..c6c6d08 --- /dev/null +++ b/judge-daemon @@ -0,0 +1,66 @@ +#!/usr/bin/env python2.6 + +import os, signal, sys, time +from resource import setrlimit, RLIMIT_AS +configs = {} +quitnow = False +judgehome = os.path.abspath(os.path.dirname(sys.argv[0])) + +def main(): + os.chdir(judgehome) + + # redirect stdout and stderr to file + os.close(1) + fd = os.open('log/daemon.log', os.O_WRONLY | os.O_APPEND | os.O_CREAT) + os.close(2) + fd = os.dup(fd) + + # setup resource usage + setrlimit(RLIMIT_AS, (1024 * 1024 * 256, 1024 * 1024 * 256)) + + # setup signal handler + signal.signal(signal.SIGINT, quit) + signal.signal(signal.SIGQUIT, quit) + + # find config files + files = os.listdir(os.path.join(judgehome, 'conf')) + for filename in files: + if filename[:5] == 'conf-' and filename[-3:] == '.py': + configs[filename] = 0 + + # start all the judge process + for conf in configs.keys(): + configs[conf] = run(conf) + + # check for the dead of judge process + while not quitnow: + for conf in configs.keys(): + pid = configs[conf] + ret = os.waitpid(pid, os.WNOHANG) + if ret[0]: + print >>sys.stderr, 'judge process %d dead, restarting...' % pid + configs[conf] = run(conf) + time.sleep(1) + +def run(conf): + pid = os.fork() + if pid: + return pid + else: + cmd = '/usr/bin/python' + conf = os.path.join('conf', conf) + os.execvp(cmd, (cmd, 'judge', conf)) + +def quit(sig, frame): + quitnow = True + for conf in configs.keys(): + pid = configs[conf] + os.kill(pid, sig) + + for conf in configs.keys(): + pid = configs[conf] + os.waitpid(pid, 0) + exit() + +if __name__ == '__main__': + main() diff --git a/judge-daemonold b/judge-daemonold new file mode 100755 index 0000000..fba9b95 --- /dev/null +++ b/judge-daemonold @@ -0,0 +1,67 @@ +#!/usr/bin/env python2.6 + +import os, signal, sys, time +from resource import setrlimit, RLIMIT_AS + +configs = {} +quitnow = False +judgehome = os.path.abspath(os.path.dirname(sys.argv[0])) + +def main(): + os.chdir(judgehome) + + # redirect stdout and stderr to file + os.close(1) + fd = os.open('log/daemon.log', os.O_WRONLY | os.O_APPEND | os.O_CREAT) + os.close(2) + fd = os.dup(fd) + + # setup resource usage + setrlimit(RLIMIT_AS, (1024 * 1024 * 256, 1024 * 1024 * 256)) + + # setup signal handler + signal.signal(signal.SIGINT, quit) + signal.signal(signal.SIGQUIT, quit) + + # find config files + files = os.listdir(os.path.join(judgehome, 'conf')) + for filename in files: + if filename[:5] == 'conf-' and filename[-3:] == '.py': + configs[filename] = 0 + + # start all the judge process + for conf in configs.keys(): + configs[conf] = run(conf) + + # check for the dead of judge process + while not quitnow: + for conf in configs.keys(): + pid = configs[conf] + ret = os.waitpid(pid, os.WNOHANG) + if ret[0]: + print >>sys.stderr, 'judge process %d dead, restarting...' % pid + configs[conf] = run(conf) + time.sleep(1) + +def run(conf): + pid = os.fork() + if pid: + return pid + else: + cmd = '/usr/bin/python2.6' + conf = os.path.join('conf', conf) + os.execvp(cmd, (cmd, 'judge', conf)) + +def quit(sig, frame): + quitnow = True + for conf in configs.keys(): + pid = configs[conf] + os.kill(pid, sig) + + for conf in configs.keys(): + pid = configs[conf] + os.waitpid(pid, 0) + exit() + +if __name__ == '__main__': + main() diff --git a/judgeold b/judgeold new file mode 100755 index 0000000..d4be3ab --- /dev/null +++ b/judgeold @@ -0,0 +1,67 @@ +#!/usr/bin/env python2.6 + +import os, signal, sys, logging, logging.handlers + +judge = None + +def setup_logging(): + from engineconfig import getConfig + config = getConfig() + + judgelog = os.path.join(config.judgehome, config.logdir, + 'judge-%s.log' % config.profile) + testerlog = os.path.join(config.judgehome, config.logdir, + 'tester-%s.log' % config.profile) + + # setup logger + # judge.main is for main judge program + logger = logging.getLogger('main') + hdlr = logging.handlers.RotatingFileHandler( + judgelog, 'a', 1024 * 1024 * 5, 4) + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + if config.debug: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + # judge.tester is for forked compiler and programs + logger = logging.getLogger('tester') + hdlr = logging.handlers.RotatingFileHandler( + testerlog, 'a', 1024 * 1024 * 5, 4) + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + if config.debug: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + +def main(): + judgehome = os.path.abspath(os.path.dirname(sys.argv[0])) + sys.path.insert(1, os.path.join(judgehome, 'python')) + + from engineconfig import getConfig + config = getConfig() + config.judgehome = judgehome + vars = {'config': config} + + execfile(sys.argv[1], vars, vars) + + setup_logging() + + from engine import JudgeEngine + judge = JudgeEngine() + + # setup signal handler + signal.signal(signal.SIGINT, judge.quit) + signal.signal(signal.SIGQUIT, judge.quit) + signal.signal(signal.SIGTERM, judge.quit) + + judge.run() + +if __name__ == '__main__': + main() + +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/python/datasource.py b/python/datasource.py new file mode 100755 index 0000000..381e5b0 --- /dev/null +++ b/python/datasource.py @@ -0,0 +1,528 @@ + +import logging, os, sys, socket, time, xmlrpclib, bz2, Cookie +import unittest +from engineconfig import getConfig +from entity import Submit, Problem, TestCase, PresetCode, DataFile + +class DataSourceError(Exception): pass + +class DataSource: + + def __init__(self, driver): + self.driver = driver + self.config = getConfig() + self.logger = logging.getLogger('main') + + self.judge_id = self.driver.get_judge_id() + + def _get_config_retry(self): + ret = self.config.retry + if ret < 1: ret = 1 + return ret + + def _get_config_retry_wait(self): + ret = self.config.retry_wait + if ret < 0.1: ret = 0.1 + return ret + + def _do_action(self, func, args): + retry = self._get_config_retry() + retry_wait = self._get_config_retry_wait() + ret = None + while retry > 0: + try: + ret = func(*args) + except DataSourceError, e: + self.logger.exception("DataSourceError") + if retry > 0: + time.sleep(retry_wait) + retry = retry - 1 + continue + else: + self.logger.info('Quit after retried.') + sys.exit(1) + break + return ret + + def get_submits(self, limit): + func = self.driver.get_submits + args = (self.judge_id, limit) + rows = self._do_action(func, args) + ret = [] + if rows: + for row in rows: + ret.append(Submit(self, row)) + return ret + + def reset_submits(self): + func = self.driver.reset_submits + args = (self.judge_id,) + return self._do_action(func, args) + + def get_problem(self, pid): + func = self.driver.get_problem + args = (pid, ) + return Problem(self, self._do_action(func, args)) + + def get_tests(self, pid, full): + func = self.driver.get_tests + args = (pid, full) + rows = self._do_action(func, args) + ret = [] + if rows: + for row in rows: + ret.append(TestCase(self, row)) + return ret + + def get_test(self, tid, raw = True): + func = self.driver.get_test + args = (tid, ) + row = self._do_action(func, args) + if raw: + return row + else: + return TestCase(self, row) + + def get_presetcodes(self, pid, lang): + func = self.driver.get_presetcodes + args = (pid, lang) + rows = self._do_action(func, args) + return map(lambda row: PresetCode(self, row), rows) + + def get_datafiles(self, pid): + func = self.driver.get_datafiles + args = (pid, ) + rows = self._do_action(func, args) + return map(lambda row: DataFile(self, row), rows) + + def get_datafile_data(self, datafileid): + func = self.driver.get_datafile_data + args = (datafileid, ) + return self._do_action(func, args) + + def update_submits_status(self, sids, status): + func = self.driver.update_submits_status + if not isinstance(sids, (tuple, list)): + sids = (sids, ) + args = (sids, status) + self.logger.debug('set submits %s status to %s', sids.__str__(), status) + return self._do_action(func, args) + + def update_submit_compilemessage(self, sid, msg): + func = self.driver.update_submit_compilemessage + args = (sid, msg) + return self._do_action(func, args) + + def get_submit_status(self, sid): + func = self.driver.get_submit_status + args = (sid, ) + return self._do_action(func, args) + + def get_submit_compilemessage(self, sid): + func = self.driver.get_submit_compilemessage + args = (sid, ) + return self._do_action(func, args) + + def update_submit_test_results(self, sid, results): + func = self.driver.update_submit_test_results + args = (sid, results) + self.logger.info('update_submits_status: submit %s, %d records' % \ + (sid, len(results))) + return self._do_action(func, args) + + def get_submit(self, sid): + func = self.driver.get_submit + args = (sid, ) + ret = self._do_action(func, args) + return Submit(self, ret) + +class JspAuthTransport(xmlrpclib.Transport): + def __init__(self): + xmlrpclib.Transport.__init__(self) + self.__cookies = Cookie.SmartCookie() + + def request(self, host, handler, request_body, verbose=0): + # issue XML-RPC request + + h = self.make_connection(host) + if verbose: + h.set_debuglevel(1) + + self.send_request(h, handler, request_body) + self.__sendJsessionCookie(h) + self.send_host(h, host) + self.send_user_agent(h) + self.send_content(h, request_body) + + errcode, errmsg, headers = h.getreply() + + if errcode != 200: + raise xmlrpclib.ProtocolError( + host + handler, + errcode, errmsg, + headers + ) + self.verbose = verbose + self.__processCookies(headers) + return self.parse_response(h.getfile()) + + + def get_jsession_id(self): + if self.__cookies.has_key('MoodleSession'): + return self.__cookies['MoodleSession'].value + return None + + + def __sendJsessionCookie(self, connection): + if self.__cookies.has_key('MoodleSession'): + connection.putheader( + 'Cookie', + 'MoodleSession=%s' % self.__cookies['MoodleSession'].value) + if self.__cookies.has_key('MoodleSessionTest'): + connection.putheader( + 'Cookie', + 'MoodleSessionTest=%s' % self.__cookies['MoodleSessionTest'].value) + + + def __processCookies(self, headers): + if headers.getheader('Set-Cookie'): + self.__cookies.load(headers.getheader('Set-Cookie')) + + + def send_content(self, connection, request_body): + connection.putheader("Content-Type", "text/xml") + connection.putheader("Content-Length", str(len(request_body))) + + connection.endheaders() + if request_body: + connection.send(request_body) + +class XmlRpcDataSource: + + def __init__(self, url): + self.transport = JspAuthTransport() + self.server = xmlrpclib.Server(url, transport = self.transport) + self.config = getConfig() + self.logger = logging.getLogger('main') + + def get_judge_id(self): + while True: + try: + return self.server.oj.get_judge_id() + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to get_judge_id') + time.sleep(self.config.retry_wait) + + def reset_submits(self, judgeid): + while True: + try: + return self.server.oj.reset_submits(judgeid) + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to reset_submits') + time.sleep(self.config.retry_wait) + + def get_submits(self, judgeid, limit): + while True: + try: + submits = self.server.oj.get_submits(judgeid, limit) + for submit in submits: + if not isinstance(submit['code'], (str, unicode)): + submit['code'] = submit['code'].__str__() + return submits + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + time.sleep(self.config.retry_wait) + self.logger.exception('Failed to get_submits') + + def get_problem(self, problemid): + while True: + try: + return self.server.oj.get_problem(problemid) + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to get_problem') + time.sleep(self.config.retry_wait) + + def get_tests(self, problemid, full): + while True: + try: + tests = self.server.oj.get_tests(problemid, full) + for test in tests: + if not isinstance(test['input'], (str, unicode)): + test['input'] = test['input'].__str__() + if not isinstance(test['output'], (str, unicode)): + test['output'] = test['output'].__str__() + self.logger.debug('Got %d test case(s)', len(tests)) + return tests + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to get_tests') + time.sleep(self.config.retry_wait) + + def get_test(self, testid): + while True: + try: + test = self.server.oj.get_gztest(testid) + if not isinstance(test['input'], (str, unicode)): + test['input'] = test['input'].__str__() + if not isinstance(test['output'], (str, unicode)): + test['output'] = test['output'].__str__() + test['input'] = bz2.decompress(test['input']) + test['output'] = bz2.decompress(test['output']) + return test + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to get_tests') + time.sleep(self.config.retry_wait) + + def get_presetcodes(self, problemid, lang): + while True: + try: + codes = self.server.oj.get_presetcodes(problemid, lang) + for code in codes: + if not isinstance(code['code'], (str, unicode)): + code['code'] = code['code'].__str__() + self.logger.debug('Got %d presetcodes', len(codes)) + return codes + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to get_presetcodes') + time.sleep(self.config.retry_wait) + + def get_datafiles(self, problemid): + while True: + try: + files = self.server.oj.get_datafiles(problemid) + self.logger.debug('Got %d datafiles', len(files)) + return files + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to get_datafiles') + time.sleep(self.config.retry_wait) + + def get_datafile_data(self, datafileid): + while True: + try: + data = self.server.oj.get_datafile_data(datafileid) + return str(data) + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to get_datafiles') + time.sleep(self.config.retry_wait) + + def update_submit_compilemessage(self, id, compilemsg): + compilemsg = xmlrpclib.Binary(compilemsg) + while True: + try: + return self.server.oj.update_submit_compilemessage( + id, compilemsg) + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to update_submit_compilemessage') + time.sleep(self.config.retry_wait) + + def update_submit_test_results(self, id, results): + for r in results: + if not isinstance(r['stdout'], (str, unicode)): r['stdout'] = '' + if not isinstance(r['stderr'], (str, unicode)): r['stderr'] = '' + r['stdout'] = xmlrpclib.Binary(r['stdout']) + r['stderr'] = xmlrpclib.Binary(r['stderr']) + while True: + try: + return self.server.oj.update_submit_test_results(id, results) + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to update_submit_compilemessage') + time.sleep(self.config.retry_wait) + + def update_submits_status(self, submitids, newstatus): + while True: + mc = xmlrpclib.MultiCall(self.server) + for id in submitids: + mc.oj.update_submit_status(id, newstatus) + try: + return mc() + except xmlrpclib.Error, e: + raise DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to update_submits_status') + time.sleep(self.config.retry_wait) + + def get_submit_status(self, sid): + while True: + try: + return self.server.oj.get_submit_status(sid) + except xmlrpclib.Error, e: + return DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to get_submit_status') + time.sleep(self.config.retry_wait) + + def get_submit_compilemessage(self, sid): + while True: + try: + msg = self.server.oj.get_submit_compilemessage(sid) + if isinstance(msg, (str, unicode)): + return msg + else: + return msg.__str__() + except xmlrpclib.Error, e: + return DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to get_submit_status') + time.sleep(self.config.retry_wait) + + def get_submit(self, sid): + while True: + try: + return self.server.oj.get_submit(sid) + except xmlrpclib.Error, e: + return DataSourceError(e) + except socket.error, e: + self.logger.exception('Failed to get_submit') + time.sleep(self.config.retry_wait) + +class DataSourceTest(unittest.TestCase): + + def setUp(self): + execfile(os.path.join('..', 'testdata', 'test_config.py')) + self.config = getConfig() + self.datasource = self.config.datasources[0] + self.dbname = os.path.join('..', 'testdata', self.config.testdb) + + + def testGetSubmits(self): + submits = self.datasource.get_submits(4) + self.assertTrue(len(submits), 4) + s = submits[0] + self.assertEqual(s.id, '0000000001') + self.assertEqual(s.problem_id, '0000000001') + self.assertEqual(s.language, 'gcc-3.3') + self.assertEqual(s.code, '') + s = submits[1] + self.assertEqual(s.code, '#include ') + + def testGetSubmit(self): + s = self.datasource.get_submit(1) + self.assertEqual(s.id, '0000000001') + self.assertEqual(s.problem_id, '0000000001') + self.assertEqual(s.language, 'gcc-3.3') + self.assertEqual(s.code, '') + + def testResetSubmits(self): + self.datasource.reset_submits() + + def testGetProblem(self): + p = self.datasource.get_problem('0000000001') + self.assertEqual(p.id, '0000000001') + + def testGetTests(self): + tests = self.datasource.get_tests('0000000001', True) + self.assertTrue(len(tests), 2) + t = tests[0] + self.assertEqual(t.id, '0000000001') + self.assertEqual(t.problem_id, '0000000001') + self.assertEqual(t.timemodified, 0) + self.assertEqual(t.input, '1 2') + self.assertEqual(t.output, '3') + self.assertEqual(t.timelimit, 10) + self.assertEqual(t.memlimit, 1024 * 1024) + + def testGetTest(self): + t = self.datasource.get_test('0000000001', False) + self.assertEqual(t.id, '0000000001') + self.assertEqual(t.problem_id, '0000000001') + self.assertEqual(t.timemodified, 0) + self.assertEqual(t.input, '1 2') + self.assertEqual(t.output, '3') + self.assertEqual(t.timelimit, 10) + self.assertEqual(t.memlimit, 1024 * 1024) + + def testUpdateSubmitCompileMessage(self): + self.datasource.update_submit_compilemessage('0000000001', 'hello') + msg = self.datasource.get_submit_compilemessage('0000000001') + self.assertEqual(msg, 'hello') + self.datasource.update_submit_compilemessage('0000000001', '') + + def testUpdateSubmitStatus(self): + self.datasource.update_submits_status(('0000000004', '0000000001'), + 'compiling') + s = self.datasource.get_submit_status('0000000001') + self.assertEqual(s, 'compiling') + s = self.datasource.get_submit_status('0000000004') + self.assertEqual(s, 'compiling') + s = self.datasource.get_submit_status('0000000002') + self.assertEqual(s, 'new') + self.datasource.update_submits_status(('0000000004', '0000000001'), 'new') + s = self.datasource.get_submit_status('0000000001') + self.assertEqual(s, 'new') + + def testUpdateSubmitTestResults(self): + from pysqlite2 import dbapi2 as sqlite + conn = sqlite.connect(self.dbname) + cur = conn.cursor() + cur.execute('DELETE FROM submit_test_result') + conn.commit() + cur.close() + conn.close() + + id = '0000000001' + r1 = { 'test_id' : '0000000001', + 'stdout' : '3', + 'stderr' : '', + 'exitcode' : 0, + 'signal' : 0, + 'timeused' : 0, + 'memused' : 0} + r2 = { 'test_id' : '0000000002', + 'stdout' : '4', + 'stderr' : 'abc', + 'exitcode' : 0, + 'signal' : 11, + 'timeused' : 1, + 'memused' : 2} + self.datasource.update_submit_test_results(id, (r1, r2)) + + conn = sqlite.connect(self.dbname) + cur = conn.cursor() + cur.execute('SELECT * FROM submit_test_result') + rows = cur.fetchall() + self.assertEqual(len(rows), 2) + cur.close() + conn.close() + row = rows[0] + self.assertEqual(row[1], 1) #sid + self.assertEqual(row[2], 1) #tid + self.assertEqual(row[3], '3') #stdout + self.assertEqual(row[4], '') #stderr + self.assertEqual(row[5], 0) #exitcode + self.assertEqual(row[6], 0) #signal + self.assertEqual(row[7], 0) #timeused + self.assertEqual(row[8], 0) #memused + row = rows[1] + self.assertEqual(row[1], 1) #sid + self.assertEqual(row[2], 2) #tid + self.assertEqual(row[3], '4') #stdout + self.assertEqual(row[4], 'abc') #stderr + self.assertEqual(row[5], 0) #exitcode + self.assertEqual(row[6], 11) #signal + self.assertEqual(row[7], 1) #timeused + self.assertEqual(row[8], 2) #memused + +if __name__ == '__main__': + unittest.main() + +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/python/datasource.pyc b/python/datasource.pyc new file mode 100644 index 0000000..f16987f Binary files /dev/null and b/python/datasource.pyc differ diff --git a/python/engine.py b/python/engine.py new file mode 100755 index 0000000..3f5838a --- /dev/null +++ b/python/engine.py @@ -0,0 +1,69 @@ + +import logging, os, thread, threading, time, Queue +from engineconfig import getConfig +class JudgeEngine: + + def __init__(self): + self.quit_event = threading.Event() + self.test_queue = Queue.Queue() + + self.config = getConfig() + self.quit_event.clear() + self.logger = logging.getLogger('main') + + def run(self): + # one thread mode is good for debugging + if self.config.test_threads == 1: + thread.start_new_thread(self.transport, ()) + self.test() + else: + for i in range(self.config.test_threads): + thread.start_new_thread(self.test, ()) + self.transport() + + for ds in self.config.datasources: + ds.reset_submits() + self.logger.info('main thread quit') + return + + def transport(self): + total = 0 + while not self.quit_event.isSet(): + self.logger.info( + "%d submit in test queue, %d processed" % \ + (self.test_queue.qsize(), total)) + c = 16 - self.test_queue.qsize() + if c > 0: + submits = self.config.datasources[0].get_submits(c) + for submit in submits: + self.logger.info('Add submit %s to queue' % submit.id) + self.test_queue.put(submit) + total = total + len(submits) + time.sleep(self.config.fetch_interval) + self.logger.info("fetch thread quit") + return + + def test(self): + while not self.quit_event.isSet(): + #pdb.set_trace() + try: + submit = self.test_queue.get(True, self.config.fetch_interval) + except Queue.Empty: + continue + tester = self.config.get_tester(submit.language) + if tester: + tester.test(submit) + else: + submit.set_status('compile_failed') + submit.set_compilemessage('No compiler specified') + self.logger.info("test thread quit") + return + + def quit(self, signum, frame): + self.logger.info('signal receive: %d' % signum) + self.quit_event.set() + return + +class JudgeError(Exception): pass + +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/python/engine.pyc b/python/engine.pyc new file mode 100644 index 0000000..7c8ea01 Binary files /dev/null and b/python/engine.pyc differ diff --git a/python/engineconfig.py b/python/engineconfig.py new file mode 100755 index 0000000..4d56a53 --- /dev/null +++ b/python/engineconfig.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python2.6 + +import string +from string import split +from os import path +from Queue import Queue +from subprocess import Popen, PIPE + +class EngineConfig: + + def __init__(self): + self.profile = 'default' + self.test_threads = 2 + self.retry = 10 + self.retry_wait = 5 + self.datasources = [] + self.fetch_interval = 1 + self.mips = 1000 + self.languages = {} + self.maxmem = 65536 + self.maxtime = 300 + self.compile_timelimit = 30 + self.output_size_limit = 1024 * 1024 * 2 + self.output_sendback_size_limit = 1024 * 64 + self.logdir = 'log' + self.debug = False + self.judgescript_wait = 10 + self.python = '/usr/bin/python3' + self.no_cleanup = False + self.datadir = path.join('data', self.profile) + self.rundir_root = '/var/lib/bitoj' + + self.runas = Queue() + self.setup_testers() + + # Init runas with ojrun?? users + pa = Popen(['getent', 'passwd'], stdout=PIPE) + pb = Popen(['awk', '-F:', '/^ojrun[0-9][0-9]:/{print $1}'], + stdin=pa.stdout, stdout=PIPE) + output = pb.communicate()[0] + if output: + for user in string.split(output, '\n'): + user = string.strip(user) + if user: self.runas.put(user) + pa.wait() + pb.wait() + + def add_tester(self, profile, tester): + self.languages[profile] = tester + + def get_tester(self, profile): + if self.languages.has_key(profile): + return self.languages[profile] + + def add_datasource(self, ds): + from datasource import DataSource + self.datasources.append(DataSource(ds)) + + def add_xmlrpc_datasource(self, url): + from datasource import XmlRpcDataSource + self.add_datasource(XmlRpcDataSource(url)) + + def setup_testers(self): + + default_compileguard = ( + '/scripts/compile-guard', ' ' + ) + default_runguard = split( + '/usr/bin/sudo -u /scripts/binary-guard ' + + '-e ' + + '-t -T 5 -m -d -o -p -x', + ' ' + ) + maxmem_runguard = split( + '/usr/bin/sudo -u /scripts/binary-guard ' + + '-e ' + + '-t -T 5 -m -d -o -p -x', + ' ' + ) + java_runguard = split( + '/usr/bin/sudo -u ' + + '/scripts/java-guard -t -T 5 -m 262144 ' + + '-e ' + + '-d -o -p -x', ' ' + ) + python_runguard = split( + '/scripts/python-guardd ' + + '-t -T 5 -m -d -o -p -x', + ' ' + ) + mono_runguard = split( + '/usr/bin/sudo -u ' + + '/scripts/mono-guard -t -T 5 -m ' + + '-e ' + + '-d -o -p -x', ' ' + ) + bash_runguard = split( + '/usr/bin/sudo -u /scripts/bash-guard ' + + '-e ' + + '-t -T 5 -m -d -o -p -x', + ' ' + ) + default_compare = ( + '/scripts/compare-guard', '', '', + '', '', '' + ) + + # Options for testers + from tester import SimpleTester, ComboTester + cbc = SimpleTester(source = 'main.c', + target = 'main', + compile = ('/scripts/gcc-3.3-bc',), + run = ('/main', ), + runenv = { + 'GCC_BOUNDS_OPTS' : '-no-message -no-statistics -no-check-mmap', + }, + basemem = {'Data' : 48, 'Stack' : 84 }, + compileguard = default_compileguard, + runguard = maxmem_runguard, + ) + + cnobc = SimpleTester( + source = 'main.c', target = 'main', + compile = ('/scripts/gcc-3.3-nobc',), + run = ('/main', ), + runenv = {}, + basemem = {'Data' : 28, 'Stack' : 84 }, + compileguard = default_compileguard, + runguard = default_runguard, + ) + + gcc33 = ComboTester() + gcc33.add_tester(cbc, has_timelimit = False, has_memlimit = False, + check_result = True, is_lasttest = False) + gcc33.add_tester(cnobc, has_timelimit = True, has_memlimit = True, + check_result = True, is_lasttest = True) + gcc33.comparecmd = default_compare + gcc33.source = 'main.c' + + gxx33 = SimpleTester( + source = 'main.cpp', target = 'main', + compile = ('/scripts/g++-3.3',), + run = ('/main',), + runenv = {}, + basemem = {'Data' : 52, 'Stack' : 84 }, + compileguard = default_compileguard, + runguard = default_runguard, + comparecmd = default_compare, + ) + + j2se15 = SimpleTester( + source = 'Main.java', target = 'Main.class', + compile = ('/scripts/javac-1.5',), + run = split('/usr/bin/java -cp -Xms8M -Xmx64M Main'), + runenv = {}, + basemem = {'RSS' : 7560 }, + baseproc = 8, + compileguard = default_compileguard, + runguard = java_runguard, + comparecmd = default_compare, + ) + + j2se16 = SimpleTester( + source = 'Main.java', target = 'Main.class', + compile = ('/scripts/javac-1.6',), + run = split('/usr/bin/java -cp -Xms8M -Xmx64M Main'), + runenv = {}, + basemem = {'RSS' : 7560 }, + baseproc = 8, + compileguard = default_compileguard, + runguard = java_runguard, + comparecmd = default_compare, + ) + + fpc = SimpleTester( + source = 'main.pas', target = 'main', + compile = ('/scripts/fpc',), + run = ('/main', ), + runenv = {}, + basemem = {'Data' : 32, 'Stack' : 84 }, + compileguard = default_compileguard, + runguard = default_runguard, + comparecmd = default_compare, + ) + + python3 = SimpleTester( + source = 'main.py', target = 'main.py', + compile = ('/bin/true',), + run = ('/usr/bin/python3', '/main.py'), + runenv = {}, + basemem = {'RSS' : 2048 }, + compileguard = (), + runguard = python_runguard, + comparecmd = default_compare, + ) + + gmcs20 = SimpleTester( + source = 'main.cs', target = 'main.exe', + compile = ('/scripts/gmcs-2.0',), + run = ('/usr/bin/mono', '/main.exe'), + runenv = { 'MONO_SHARED_DIR' : '' }, + basemem = {'RSS' : 8192 }, + compileguard = (), + runguard = mono_runguard, + comparecmd = default_compare, + ) + + bash3 = SimpleTester( + source = 'main.sh', target = 'main.sh', + compile = ('/bin/true',), + run = ('/bin/bash', '/main.sh'), + runenv = {}, + basemem = {'RSS' : 2048 }, + extraproc = 3, + compileguard = (), + runguard = bash_runguard, + comparecmd = default_compare, + ) + + self.add_tester('gcc-3.3', gcc33) + self.add_tester('gcc-3.3-nobc', cnobc) + self.add_tester('gcc-3.3-bc', cbc) + self.add_tester('g++-3.3', gxx33) + self.add_tester('java-1.5', j2se15) + self.add_tester('java-1.6', j2se16) + self.add_tester('fpc-2.2', fpc) + self.add_tester('python3', python3) + self.add_tester('gmcs-2.0', gmcs20) + self.add_tester('bash-3', bash3) + +config = None + +def getConfig(): + global config + if not config: + config = EngineConfig() + return config + +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/python/engineconfig.pyc b/python/engineconfig.pyc new file mode 100644 index 0000000..95948e4 Binary files /dev/null and b/python/engineconfig.pyc differ diff --git a/python/entity.py b/python/entity.py new file mode 100755 index 0000000..bee3ce2 --- /dev/null +++ b/python/entity.py @@ -0,0 +1,271 @@ + +import string, os, stat, threading, logging, bz2 +import unittest + +from engineconfig import getConfig +from judgescript import InternalJudge, ExternalJudge + +class Problem: + + def __init__(self, datasource, row): + self.tests = None + self.presetcodes = {} + self.datafiles = None + + self.datasource = datasource + self.id = row['id'] + self.timemodified = row['timemodified'] + self.vcode = row['validator_code'] + if not isinstance(self.vcode, (str, unicode)): + self.vcode = self.vcode.__str__() + self.vtype = row['validator_type'] + self.vlang = row['validator_lang'] + self.gcode = row['generator_code'] + if not isinstance(self.gcode, (str, unicode)): + self.gcode = self.vcode.__str__() + self.gtype = row['generator_type'] + self.standard_code = row['standard_code'] + + if row.has_key('input_filename') and row['input_filename']: + self.input_filename = row['input_filename'] + else: + self.input_filename = None + if row.has_key('output_filename') and row['output_filename']: + self.output_filename = row['output_filename'] + else: + self.output_filename = None + + def get_judge_script(self): + if self.vtype == 'comparetext': + return InternalJudge() + elif self.vtype == 'comparetextwithpe': + return InternalJudge(allowpe = True) + elif self.vtype == 'comparefile': + code = open('scripts/compare-file.py').read() + return ExternalJudge(self.id, 'python-2.5', code) + elif self.vtype == 'customized': + return ExternalJudge(self.id, self.vlang, self.vcode) + + def get_testcases(self): + if not self.tests: + self.tests = self.datasource.get_tests(self.id, False) + for testcase in self.tests: + testcase.problem = self + return self.tests + + def get_input_filename(self): + return self.input_filename + + def get_output_filename(self): + return self.output_filename + + def get_presetcodes(self, lang): + if not self.presetcodes.has_key(lang): + codes = self.datasource.get_presetcodes(self.id, lang) + for code in codes: code.problem = self + self.presetcodes[lang] = codes + return self.presetcodes[lang] + + def get_datafiles(self): + if not self.datafiles: + self.datafiles = self.datasource.get_datafiles(self.id) + return self.datafiles + +class Submit: + + def __init__(self, datasource, row): + self.datasource = datasource + self.problem = None + self.id = row['id'] + self.problem_id = row['problem_id'] + self.language = row['language'] + self.code = row['code'] + + def get_problem(self): + if not self.problem: + self.problem = self.datasource.get_problem(self.problem_id) + return self.problem + + def get_testcases(self): + return self.get_problem().get_testcases() + + def get_judge_script(self): + return self.get_problem().get_judge_script() + + def get_input_filename(self): + return self.get_problem().get_input_filename() + + def get_output_filename(self): + return self.get_problem().get_output_filename() + + def get_presetcodes(self): + return self.get_problem().get_presetcodes(self.language) + + def set_compilemessage(self, msg): + return self.datasource.update_submit_compilemessage(self.id, msg) + + def set_status(self, newstatus): + return self.datasource.update_submits_status(self.id, newstatus) + + def get_status(self): + return self.datasource.get_submit_status(self.id) + + def get_compilemessage(self): + return self.datasource.get_submit_compilemessage(self.id) + + def update_test_results(self, results): + config = getConfig() + + newresults = [] + for r in results: + # stdout + f = file(r[4], 'r') + r[4] = f.read(config.output_sendback_size_limit) + f.close() + # stderr + f = file(r[5], 'r') + r[5] = f.read(config.output_sendback_size_limit) + f.close() + + # strip stdout and stderr send back to datasource + # preventing post data too big + if len(r[4]) > config.output_sendback_size_limit: + r[4] = r[4][0:config.output_sendback_size_limit] + if len(r[5]) > config.output_sendback_size_limit: + r[5] = r[5][0:config.output_sendback_size_limit] + + newresults.append({ 'test_id' : r[0], + 'judge_result' : r[1], + 'exitcode' : r[2], + 'signal' : r[3], + 'stdout' : r[4], + 'stderr' : r[5], + 'timeused' : r[6], + 'memused' : r[7] }) + + return self.datasource.update_submit_test_results(self.id, newresults) + +class TestCase: + + write_lock = threading.Lock() + + def __init__(self, datasource, row): + config = getConfig() + logger = logging.getLogger('main') + self.id = row['id'] + self.problem_id = row['problem_id'] + self.timemodified = row['timemodified'] + #self.input = string.replace(row['input'], '\r\n', '\n') + #self.output = string.replace(row['output'], '\r\n', '\n') + self.timelimit = row['timelimit'] + if not self.timelimit: + self.timelimit = 1 + self.memlimit = row['memlimit'] + if not self.memlimit: + self.memlimit = config.maxmem + if row.has_key('nproc') and row['nproc']: + self.nproc = string.atoi(row['nproc']) + else: + self.nproc = 0 + + config = getConfig() + testdir = os.path.join(config.datadir, 'testcase') + if not os.path.exists(testdir): os.mkdir(testdir) + + self.infile = os.path.join(testdir, self.id + '.in') + self.outfile = os.path.join(testdir, self.id + '.out') + + TestCase.write_lock.acquire() + + if os.path.exists(self.infile): + inmtime = os.stat(self.infile)[stat.ST_MTIME] + else: + inmtime = 0 + if os.path.exists(self.outfile): + outmtime = os.stat(self.outfile)[stat.ST_MTIME] + else: + outmtime = 0 + logger.debug("inmtime=%d outmtime=%d timemodified=%d" % (inmtime, outmtime, self.timemodified)) + if inmtime <= self.timemodified or outmtime <= self.timemodified: + logger.debug('Creating input/output file %s and %s' % (self.infile, self.outfile)) + row = datasource.get_test(self.id) + input = string.replace(row['input'], '\r\n', '\n') + output = string.replace(row['output'], '\r\n', '\n') + + f = file(self.infile, 'w') + f.write(input) + if len(input) > 0 and input[-1] != '\n': f.write('\n') + f.close() + + f = file(self.outfile, 'w') + f.write(output) + if len(output) > 0 and output[-1] != '\n': f.write('\n') + f.close() + logger.debug('Finished') + else: + logger.debug('Skip input/output file creation') + + TestCase.write_lock.release() + + def get_input_file(self): + return self.infile + + def get_output_file(self): + return self.outfile + +class PresetCode: + + def __init__(self, datasource, row): + self.datasource = datasource + self.problem = None + + self.id = row['id'] + self.name = row['name'] + self.code = row['code'] + self.isheader = row['isheader'] + +class DataFile: + + write_lock = threading.Lock() + + def __init__(self, datasource, row): + self.datasource = datasource + self.problem = None + + # Data passed from datasource + self.id = row['id'] + self.problem_id = row['problem_id'] + self.filename = row['filename'] + self.type = row['type'] + self.timemodified = row['timemodified'] + + # Save datafile + config = getConfig() + testdir = os.path.join(config.datadir, 'testcase') + if not os.path.exists(testdir): os.mkdir(testdir) + self.absolute_path = os.path.join( + testdir, self.problem_id + '.' + self.filename) + + DataFile.write_lock.acquire() + + mtime = 0 + if os.path.exists(self.absolute_path): + mtime = os.stat(self.absolute_path)[stat.ST_MTIME] + if mtime < self.timemodified: + data = datasource.get_datafile_data(self.id) + data = bz2.decompress(data) + if self.type == 'text': + f = open(self.absolute_path, 'w') + f.write(string.replace(data, '\r\n', '\n')) + f.close() + else: + f = open(self.absolute_path, 'wb') + f.write(data) + f.close() + + DataFile.write_lock.release() + +if __name__ == '__main__': + unittest.main() + +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/python/entity.pyc b/python/entity.pyc new file mode 100644 index 0000000..ef16f4c Binary files /dev/null and b/python/entity.pyc differ diff --git a/python/judgescript.py b/python/judgescript.py new file mode 100755 index 0000000..7a2a6ec --- /dev/null +++ b/python/judgescript.py @@ -0,0 +1,286 @@ + +import string, os, time, tempfile, logging +import unittest + +from engineconfig import getConfig + +class InternalJudge: + + def __init__(self, allowpe = False): + self.logger = logging.getLogger('main') + self.allowpe = allowpe + + def judge(self, sid, tid, tin, tout, result, errfile, rundir = None): + """Judge if the result correct. + + @param sid String submit id + @param tid String testcase id + @param tin String filename of the standard input + @param tout String filename of the standard output + @param result String filename of the result + """ + + return self.compare_file(tin, tout, result, self.allowpe) + + def compare_file(self, input, output, result, allowpe): + fo = file(output, 'rb'); fr = file(result, 'rb') + + if not allowpe: + r = 'AC' + while r == 'AC': + so = fo.read(8192); sr = fr.read(8192) + if so == '' and sr == '': break + if so != sr: r = 'WA' + else: + so = fo.read(); sr = fr.read() + r = self.compare_string(so, sr) + + fo.close(); fr.close() + return r + + def compare_string(self, output, result): + if output == result: return 'AC' + + outnum = '' + retnum = '' + for c in output: + if c in string.digits: outnum += c + for c in result: + if c in string.digits: retnum += c + self.logger.debug('numbers in output: %s' % outnum) + self.logger.debug('numbers in result: %s' % retnum) + if len(outnum) > 0 and len(retnum) > 0 and outnum == retnum: + return 'PE' + + return 'WA' + +class ExternalJudge: + + def __init__(self, problemid, vlang, vcode): + """Constructor. + + @param pid String problem id + @param vlang String validator language + @param vcode String validator code + """ + self.config = getConfig() + self.logger = logging.getLogger('main') + self.problemid = problemid + self.lang = vlang + self.code = vcode + + # save validator code + tester = self.config.get_tester(self.lang) + datadir = os.path.join(self.config.datadir, 'validator', problemid) + if not os.path.exists(datadir): + os.makedirs(datadir) + self.comparecmd = tester.comparecmd + self.codefile = os.path.abspath(os.path.join(datadir, tester.source)) + f = file(self.codefile, 'w') + f.write(string.replace(vcode, '\r\n', '\n')) + if len(vcode) > 0 and vcode[-1] != '\n': f.write('\n') + f.close() + + self.logger.debug("Save validator code as %s" % self.codefile) + + def judge(self, sid, tid, tin, tout, result, errfile, rundir = None): + """Judge if the result correct. + + @param sid String submit id + @param tid String testcase id + @param tin String filename of the standard input + @param tout String filename of the standard output + @param result String filename of the result + @param rundir String in which dir the judge script should be started + """ + + rfiledir = os.path.join(self.config.datadir, sid) + rfile = os.path.join(rfiledir, tid + '.rst') + if not os.path.exists(rfiledir): + os.mkdir(rfiledir) + + cmd = [] + for s in self.comparecmd: + s = s.replace('', self.config.judgehome) + s = s.replace('', self.lang) + s = s.replace('', self.codefile) + s = s.replace('', tin) + s = s.replace('', tout) + s = s.replace('', result) + cmd.append(s) + self.logger.debug("Run validator as %s" % cmd.__str__()) + + pid = os.fork() + if pid == 0: + os.close(1) + os.open(rfile, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0666) + os.close(2) + os.open(errfile, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0666) + + if rundir: os.chdir(rundir) + os.execv(cmd[0], cmd) + print 'WA' + + remaintime = self.config.judgescript_wait + pid1 = 0 + while remaintime > 0: + pid1, status = os.waitpid(pid, os.WNOHANG) + if pid1 > 0: + break + time.sleep(0.1) + remaintime -= 0.1 + + while pid1 == 0: + try: + os.kill(pid, 9) + except os.OSError, e: + pass + pid1, status = os.waitpid(pid, os.WNOHANG) + + f = file(rfile, 'r') + ret = string.strip(f.readline()) + f.close() + if ret != 'AC' and ret != 'PE': ret = 'WA' + return ret + + +class InternalJudgeTest(unittest.TestCase): + """Internal Judge Test Case.""" + + def testWithoutPE(self): + ij = InternalJudge() + output = 'hello world\n' + result = 'hello world\n' + self.assertEqual(ij.compare_string(output, result), 'AC') + output = 'hello world' + self.assertEqual(ij.compare_string(output, result), 'WA') + result = 'hello world' + self.assertEqual(ij.compare_string(output, result), 'AC') + result = 'hello world ' + self.assertEqual(ij.compare_string(output, result), 'WA') + + def testWithPE(self): + ij = InternalJudge(True) + input = '' + output = 'hello world\n' + result = 'hello world\n' + self.assertEqual(ij.compare_string(output, result), 'AC') + result = 'hello worl\n' + self.assertEqual(ij.compare_string(output, result), 'WA') + output = '1 2 3 4 5\n' + result = '1\n2\n3\n4\n5\n' + self.assertEqual(ij.compare_string(output, result), 'PE') + result = '12345\n' + self.assertEqual(ij.compare_string(output, result), 'PE') + result = '1 2 3 4 5' + self.assertEqual(ij.compare_string(output, result), 'PE') + +class ExternalJudgeTest(unittest.TestCase): + """External Judge Test Case.""" + + def setUp(self): + config = getConfig() + config.judgehome = '../' + config.datadir = os.path.join('..', 'testdata') + + self.tin = tempfile.NamedTemporaryFile('w') + self.tin.write('3') + self.tin.flush() + self.tout = tempfile.NamedTemporaryFile('w') + self.tout.write('3') + self.tout.flush() + self.rst = tempfile.NamedTemporaryFile('w') + self.rst.write('6') + self.rst.flush() + self.err = tempfile.NamedTemporaryFile('w') + self.err.close() + self.tin.close() + self.tout.close() + self.rst.close() + + def testGCC33(self): + code = """ +#include + +int main() { + printf("AC\\n"); + return 0; +} +""" + self.j = ExternalJudge('p1', 'gcc-3.3', code) + x = self.j.judge('s1', 't1', self.tin.name, self.tout.name, self.rst.name, self.err.name) + self.assertEquals('AC', x) + + def testGPP33(self): + code = """ +#include +using namespace std; +int main() +{ + cout << "AC" << endl; + return 0; +} +""" + self.j = ExternalJudge('p1', 'g++-3.3', code) + x = self.j.judge('s1', 't1', self.tin.name, self.tout.name, self.rst.name, self.err.name) + self.assertEquals('AC', x) + + def testFPC(self): + code = """ +program main; + +begin + write('AC'); +end. +""" + self.j = ExternalJudge('p1', 'fpc-2.2', code) + x = self.j.judge('s1', 't1', self.tin.name, self.tout.name, self.rst.name, self.err.name) + self.assertEquals('AC', x) + + def testJava(self): + code = """ +public class Main { + public static void main(String[] args) { + System.out.println("AC"); + } +} +""" + self.j = ExternalJudge('p1', 'java-1.5', code) + x = self.j.judge('s1', 't1', self.tin.name, self.tout.name, self.rst.name, self.err.name) + self.assertEquals('AC', x) + self.j = ExternalJudge('p1', 'java-1.6', code) + x = self.j.judge('s1', 't1', self.tin.name, self.tout.name, self.rst.name, self.err.name) + self.assertEquals('AC', x) + + def testPython(self): + code = """public class Main +{ + public static void Main() + { + System.Console.WriteLine("AC"); + } +} +""" + self.j = ExternalJudge('p1', 'gmcs-2.0', code) + x = self.j.judge('s1', 't1', self.tin.name, self.tout.name, self.rst.name, self.err.name) + self.assertEquals('AC', x) + + def testPython(self): + code = """#!/usr/bin/env python + +print 'AC' +""" + self.j = ExternalJudge('p1', 'python-2.5', code) + x = self.j.judge('s1', 't1', self.tin.name, self.tout.name, self.rst.name, self.err.name) + self.assertEquals('AC', x) + + def testBash(self): + code = """echo AC""" + self.j = ExternalJudge('p1', 'bash-3', code) + x = self.j.judge('s1', 't1', self.tin.name, self.tout.name, self.rst.name, self.err.name) + self.assertEquals('AC', x) + +if __name__ == '__main__': + unittest.main() + +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/python/judgescript.pyc b/python/judgescript.pyc new file mode 100644 index 0000000..4a8bba8 Binary files /dev/null and b/python/judgescript.pyc differ diff --git a/python/ojunit.py b/python/ojunit.py new file mode 100755 index 0000000..bf70843 --- /dev/null +++ b/python/ojunit.py @@ -0,0 +1,21 @@ +import unittest, logging, os, sys + +from engineconfig import getConfig + +class OJTestCase(unittest.TestCase): + + def setUp(self): + + logger = logging.getLogger('main') + hdlr = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + hdlr.setFormatter(formatter) + logger.addHandler(hdlr) + logger.setLevel(logging.DEBUG) + + execfile(os.path.join('..', 'testdata', 'test_config.py')) + self.config = getConfig() + self.ds = self.config.datasources[0] + self.dbname = os.path.join('..', 'testdata', self.config.testdb) + + self.logger = logger diff --git a/python/ojunit.pyc b/python/ojunit.pyc new file mode 100644 index 0000000..7204ec3 Binary files /dev/null and b/python/ojunit.pyc differ diff --git a/python/tester.py b/python/tester.py new file mode 100755 index 0000000..1233a08 --- /dev/null +++ b/python/tester.py @@ -0,0 +1,829 @@ + +import math, os, resource, signal, string, sys, threading, logging, time, pickle +import shutil, Queue +import unittest +import pdb +from engineconfig import getConfig +from ojunit import * + +class TesterBase: + + def __init__(self): + self.logger = logging.getLogger('main') + + def get_datadir(self, submit): + config = getConfig() + + # get datadir + datadir = os.path.abspath(config.datadir) + if not os.path.exists(datadir): os.mkdir(datadir) + + # get filenames + datadir = os.path.join(datadir, submit.id) + if not os.path.exists(datadir): os.mkdir(datadir) + + return datadir + + def get_rundir(self, submit): + config = getConfig() + return os.path.join(config.rundir_root, submit.user, 'run') + +class SimpleTester(TesterBase): + + has_timelimit = True + has_memlimit = True + check_result = True + is_lasttest = True + + def __init__(self, + source, target, + compile = None, compileenv = {}, + run = None, runenv = {}, basemem = 0, + baseproc = 0, extraproc = 0, + compileguard = None, runguard = None, comparecmd = None): + TesterBase.__init__(self) + self.source = source + self.target = target + self.compilecmd = compile + self.compileenv = compileenv + self.runcmd = run + self.runenv = runenv + self.basemem = basemem + self.baseproc = baseproc + self.extraproc = extraproc + self.compileguard = compileguard + self.runguard = runguard + self.comparecmd = comparecmd + + def test(self, submit): + #pdb.set_trace() + if not self.prepare(submit): + self.logger.info('self.prepare') + submit.set_status('compile_failed') + submit.set_compilemessage('Failed to delete source/target file') + return False + + submit.set_status('compiling') + if not self.compile(submit): + self.logger.info('self.compile') + submit.set_status('compile_failed') + self.cleanup(submit) + return False + + submit.set_status('running') + testcases = submit.get_testcases() + ret = [] + for testcase in testcases: + r = self.run(submit, testcase) + ret.append(r) + self.logger.debug('run result: ' + r.__str__()) + + passed = True + if self.check_result: + for r in ret: + if r[1] == 'AC' or not self.has_timelimit and r[1] == 'TLE': + pass + else: + passed = False + if not passed or self.is_lasttest: + submit.update_test_results(ret) + submit.set_status('finish') + + self.cleanup(submit) + + return passed + + def prepare(self, submit): + datadir = self.get_datadir(submit) + + # write presetcodes to dir + presetcodes = submit.get_presetcodes() + for presetcode in presetcodes: + pcname = os.path.join(datadir, presetcode.name) + if os.path.exists(pcname): + try: + os.unlink(pcname) + except OSError, e: + self.logger.exception( + "Failed to delete presetcode file %s" % pcname) + return False + f = open(pcname, 'w') + f.write(string.replace(presetcode.code, '\r\n', '\n')) + f.write('\n'); + f.close() + + # delete existing source and target file + datadirsource = os.path.join(datadir, self.source) + datadirtarget = os.path.join(datadir, self.target) + if os.path.exists(datadirsource): + try: + os.unlink(datadirsource) + except OSError, e: + self.logger.exception("Failed to delete source") + return False + if os.path.exists(datadirtarget): + try: + os.unlink(datadirtarget) + except OSError, e: + self.logger.exception("Failed to delete target") + return False + + # preprocess source code + code = string.replace(submit.code, '\r\n', '\n') + code = string.replace(code, chr(0x1a), '') # char generated by tc + code = string.replace(code, 'getch()', '') + code = string.replace(code, 'getch ()', '') + code = string.replace(code, 'getch ( )', '') + + code = string.replace(code, '\r\n', '\n') + # write source to disk + f = open(datadirsource, 'w') + f.write(code) + if len(submit.code) > 0 and submit.code[-1] != '\n': f.write('\n') + f.close() + + # setup rundir + config = getConfig() + try: + submit.user = config.runas.get_nowait() + except Queue.Empty: + self.logger.exception("No runas user left, please create more!") + return False + rundir = self.get_rundir(submit) + + if not os.path.exists(os.path.dirname(rundir)): + os.mkdir(os.path.dirname(rundir)) + + if os.path.exists(rundir): + try: + self._remove(rundir) + except OSError, e: + self.logger.exception("Failed to delete rundir") + config.runas.put(submit.user) + return False + os.mkdir(rundir) + os.chmod(rundir, 0775) + + return True + + def cleanup(self, submit, force = False): + datadir = self.get_datadir(submit) + rundir = self.get_rundir(submit) + config = getConfig() + if not config.no_cleanup or force: + if os.path.exists(datadir): + self._remove(datadir) + if os.path.exists(rundir): + try: + self._remove(rundir) + except OSError: + os.system('/usr/bin/sudo -u %s /bin/rm -rf %s' % (submit.user, rundir)) + + config.runas.put(submit.user) + + def compile(self, submit): + config = getConfig() + datadir = self.get_datadir(submit) + cmd = [] + if self.compileguard: + for s in self.compileguard: + s = string.replace(s, '', config.judgehome) + s = string.replace(s, '', datadir) + cmd.append(s) + if self.compilecmd: + for s in self.compilecmd: + s = string.replace(s, '', config.judgehome) + s = string.replace(s, '', datadir) + cmd.append(s) + cmd.append(self.source) + for code in submit.get_presetcodes(): + if not code.isheader: + cmd.append(code.name) + self.logger.debug(string.join(cmd, '_')) + + errfile = os.path.join(datadir, 'compile.err') + + (exitcode, sig, timeused, memused) = \ + self._execute(submit, cmd, timelimit = config.compile_timelimit, + infile = None, outfile = None, errfile = errfile, + env = self.compileenv) + + compilemsg = None + if os.path.exists(errfile): + f = file(errfile, 'r') + compilemsg = string.join(f.readlines(), '') + f.close() + + if compilemsg: + submit.set_compilemessage(compilemsg) + + if exitcode == 0: + self.logger.info('submit %s compile success' % submit.id) + else: + self.logger.info('submit %s compile failed' % submit.id) + self.logger.debug(compilemsg) + + return exitcode == 0 + + def run(self, submit, testcase): + config = getConfig() + datadir = self.get_datadir(submit) #dir save input and result + rundir = self.get_rundir(submit) #dir program runs in + + infile = testcase.get_input_file() + outfile = os.path.join(datadir, testcase.id + '.out') + errfile = os.path.join(datadir, testcase.id + '.err') + statfile = os.path.join(rundir, testcase.id + '.stat') + + # If submit input filename or output filename is provided, + # no input or output redirect will happen. + submit_input_filename = submit.get_input_filename() + submit_output_filename = submit.get_output_filename() + if submit_input_filename: + shutil.copyfile(infile, os.path.join(rundir, submit_input_filename)) + _infile = infile + infile = None + if submit_output_filename: + _outfile = outfile + outfile = None + + # Create data file symolic links + datafiles = submit.get_problem().get_datafiles() + for datafile in datafiles: + targetname = os.path.join(rundir, datafile.filename) + try: + shutil.copyfile(datafile.absolute_path, targetname) + except: + pass + + # Use extra process setting on testcase if it exist + extraproc = self.baseproc + (testcase.nproc if testcase.nproc else self.extraproc) + + cmd = [] + if self.runguard: + for s in self.runguard: + s = s.replace('', config.judgehome) + s = s.replace('', '%d' % extraproc) + s = s.replace('', '%d' % (testcase.timelimit)) + s = s.replace('', '%d' % config.maxmem) + s = s.replace('', rundir) + s = s.replace('', submit.user) + s = s.replace('', statfile) + cmd.append(s) + if self.runcmd: + for s in self.runcmd: + s = s.replace('', config.judgehome) + s = s.replace('', datadir) + s = s.replace('', submit.user) + cmd.append(s) + if self.runenv: + for k in self.runenv.keys(): + s = self.runenv[k] + s = s.replace('', config.judgehome) + s = s.replace('', datadir) + s = s.replace('', submit.user) + self.runenv[k] = s + self.logger.debug(string.join(cmd, ' ') + ' ' + str(self.runenv)) + + (exitcode, sig, timeused, memused) = \ + self._execute(submit, cmd, timelimit = testcase.timelimit * 10, + infile = infile, outfile = outfile, + errfile = errfile, env = self.runenv, + rlimit_fsize = config.output_size_limit + 2, + statfile = statfile) + if submit_input_filename: + infile = _infile + if submit_output_filename: + outfile = _outfile + try: + shutil.copyfile(os.path.join(rundir, submit_output_filename), + outfile) + except IOError: + f = file(outfile, 'w') + f.close() + + ret = [testcase.id, exitcode, sig, outfile, errfile, timeused, memused] + if timeused > testcase.timelimit: + ret.insert(1, 'TLE') + return ret + if memused > testcase.memlimit: + ret.insert(1, 'MLE') + return ret + + if exitcode == 125: + ret.insert(1, 'JSE') + return ret + elif exitcode == 126: + ret.insert(1, 'RFC') + return ret + elif exitcode == 127: + ret.insert(1, 'JGE') + return ret + + if sig == signal.SIGABRT or sig == signal.SIGSEGV: + ret.insert(1, 'RE') + return ret + elif sig == signal.SIGKILL or sig == signal.SIGXCPU: + ret.insert(1, 'TLE') + return ret + elif sig == signal.SIGXFSZ: + ret.insert(1, 'OLE') + return ret + elif sig == signal.SIGFPE: + ret.insert(1, 'FPE') + return ret + elif sig > 0: + ret.insert(1, 'KS') + return ret + + s = os.stat(outfile) + if s.st_size > config.output_size_limit: + ret.insert(1, 'OLE') + return ret + + js = submit.get_judge_script() + ret.insert(1, js.judge(submit.id, testcase.id, + os.path.abspath(testcase.get_input_file()), + os.path.abspath(testcase.get_output_file()), + os.path.abspath(outfile), + os.path.abspath(errfile), rundir)) + return ret + + def _execute(self, submit, command, timelimit = 1, + infile = None, outfile = None, errfile = None, env = {}, + rlimit_fsize = -1, statfile = None): + + pid = os.fork() + + if pid == 0: + if rlimit_fsize > 0: + resource.setrlimit(resource.RLIMIT_FSIZE, + (rlimit_fsize, rlimit_fsize)) + + # child process + if infile: + try: + os.close(0) + os.open(infile, os.O_RDONLY) + except Exception, e: + print e + sys.exit(125) + if outfile: + try: + os.close(1) + os.open(outfile, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0666) + except Exception, e: + print e + sys.exit(125) + if errfile: + try: + os.close(2) + os.open(errfile, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0666) + except Exception, e: + print e + sys.exit(125) + + #os.chdir(self.get_datadir(submit)) + os.execve(command[0], command, env) + sys.exit(125) + + # parent process + pid, status = os.waitpid(pid, 0) + timeused = 0; memused = 0; sig = 0; exitcode = 0 + + # get this child sig and exitcode + if os.WIFEXITED(status): exitcode = os.WEXITSTATUS(status) + if os.WIFSIGNALED(status): sig = os.WTERMSIG(status) + + # read information form statfile + if statfile: + try: + stat = pickle.load(file(statfile, 'r')) + exitcode = stat['exitcode'] + sig = stat['sig'] + timeused = stat['timeused'] + memused = 0 + if self.basemem.has_key('RSS'): + memused += stat['memrss'] - self.basemem['RSS'] + if self.basemem.has_key('Data'): + memused += stat['memdata'] - self.basemem['Data'] + if self.basemem.has_key('Stack'): + memused += stat['memstack'] - self.basemem['Stack'] + memused = max(0, memused) + except Exception, e: + self.logger.exception(e) + self.logger.error("Failed to read statfile: %s" % statfile) + exitcode = 127 # judge script error + + return (exitcode, sig, timeused, memused) + + def _remove(self, top): + for root, dirs, files in os.walk(top, topdown=False): + for name in files: + os.remove(os.path.join(root, name)) + for name in dirs: + os.rmdir(os.path.join(root, name)) + os.rmdir(top) + +class ComboTester: + + testers = [] + + def add_tester(self, tester, + has_timelimit = True, has_memlimit = True, + check_result = True, is_lasttest = False): + tester.has_timelimit = has_timelimit + tester.has_memlimit = has_memlimit + tester.check_result = check_result + tester.is_lasttest = is_lasttest + self.testers.append(tester) + + def test(self, submit): + r = True + for t in self.testers: + r = t.test(submit) + if not r: + break + return r + +class SimpleTesterTestCase(OJTestCase): + + @classmethod + def suite(clazz): + tests = ( 'testSteps', 'testNoSource', + 'testAC', 'testWA', 'testZero', + 'testBinary', 'testCE', 'testOLE', 'testTLE', + 'testTargetDeleted', + ) + return unittest.TestSuite(map(SimpleTesterTestCase, tests)) + + def setUp(self): + OJTestCase.setUp(self) + self.st = self.config.languages['gcc-3.3-nobc'] + + def testSteps(self): + self.logger.info("TESTING: Steps") + submit = self.ds.get_submit(3) + + self.st.prepare(submit) + datadir = os.path.join(self.config.datadir, submit.id) + self.assertTrue(os.path.isdir(datadir)) + self.assertTrue(os.path.isfile(os.path.join(datadir, self.st.source))) + + submit.set_status('compiling') + r = self.st.compile(submit) + self.assertTrue(r) + self.assertTrue(os.path.isfile(os.path.join(datadir, self.st.target))) + self.assertTrue(os.path.isfile(os.path.join(datadir, 'compile.err'))) + + submit.set_status('running') + testcases = submit.get_testcases() + + r = self.st.run(submit, testcases[0]) + self.assertEqual(r[0], testcases[0].id) + self.assertEqual(r[1], 'AC') + self.assertEqual(r[2], 0) + self.assertEqual(r[3], 0) + f = file(r[4], 'r') + o = string.join(f.readlines(), '\n') + f.close() + self.assertEqual(o, '3\n') + f = file(r[5], 'r') + o = string.join(f.readlines(), '\n') + f.close() + self.assertEqual(o, '') + + r = self.st.run(submit, testcases[1]) + f = file(r[4], 'r') + o = string.join(f.readlines(), '\n') + f.close() + self.assertEqual(o, '4\n') + + self.st.cleanup(submit) + self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir)) + + def testAC(self): + self.logger.info("TESTING: AC") + submit = self.ds.get_submit(3) + + datadir = os.path.join(self.config.datadir, submit.id) + r = self.st.test(submit) + self.assertTrue(r) # passed + + self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir)) # cleanup + + from pysqlite2 import dbapi2 as sqlite + conn = sqlite.connect(self.dbname) + cur = conn.cursor() + + cur.execute('SELECT * FROM submit_test_result WHERE sid=?', (submit.id, )) + rows = cur.fetchall() + cur.execute('DELETE FROM submit_test_result') + conn.commit() + + self.assertEqual(len(rows), 2) + + # test if the submit status change + cur.execute('SELECT status FROM submit WHERE id=?', (submit.id, )) + rows = cur.fetchall() + self.assertEqual(rows[0][0], 10) + + cur.close() + conn.close() + + def testNoSource(self): + self.logger.info("TESTING: No Source") + submit = self.ds.get_submit(1) + + # the code of this submit is empty + datadir = os.path.join(self.config.datadir, submit.id) + r = self.st.test(submit) + self.assertFalse(r) + self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir)) # cleanup + self.assertTrue(submit.get_status(), 'compile_failed') + + def testNoTarget(self): + self.logger.info("TESTING: No Target") + submit = self.ds.get_submit(1) + + # the code of this submit generate nothing + datadir = os.path.join(self.config.datadir, submit.id) + r = self.st.test(submit) + self.assertFalse(r) + self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir)) # cleanup + self.assertTrue(submit.get_status(), 'compile_failed') + + def testCE(self): + self.logger.info("TESTING: CE") + submit = self.ds.get_submit(7) + + datadir = os.path.join(self.config.datadir, submit.id) + r = self.st.test(submit) + self.assertFalse(r) # not passed + + self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir)) # cleanup + + self.assertTrue(len(submit.get_compilemessage()) > 0) + self.assertTrue(submit.get_status(), 'compile_failed') + + def testWA(self): + self.logger.info("TESTING: WA") + submit = self.ds.get_submit(4) + + datadir = os.path.join(self.config.datadir, submit.id) + r = self.st.test(submit) + self.assertFalse(r) # not passed + self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir)) # cleanup + from pysqlite2 import dbapi2 as sqlite + conn = sqlite.connect(self.dbname) + cur = conn.cursor() + + cur.execute('SELECT * FROM submit_test_result') + rows = cur.fetchall() + cur.execute('DELETE FROM submit_test_result') + conn.commit() + + self.assertEqual(len(rows), 2) + + cur.close() + conn.close() + + def testTargetDeleted(self): + self.logger.info("TESTING: Target Deleted") + submit = self.ds.get_submit(3) + + self.st.prepare(submit) + datadir = os.path.join(self.config.datadir, submit.id) + self.assertTrue(os.path.isdir(datadir)) + self.assertTrue(os.path.isfile(os.path.join(datadir, self.st.source))) + + submit.set_status('compiling') + r = self.st.compile(submit) + self.assertTrue(r) + self.assertTrue(os.path.isfile(os.path.join(datadir, self.st.target))) + self.assertTrue(os.path.isfile(os.path.join(datadir, 'compile.err'))) + + self.st.cleanup(submit, True) + + submit.set_status('running') + testcases = submit.get_testcases() + r = self.st.run(submit, testcases[0]) + self.assertNotEqual(r[1], 'AC') + + def testZero(self): + self.logger.info("TESTING: ZERO") + submit = self.ds.get_submit(5) + + # this program output '\0' + datadir = os.path.join(self.config.datadir, submit.id) + self.st.prepare(submit) + r = self.st.compile(submit) + self.assertTrue(r) + testcases = submit.get_testcases() + r = self.st.run(submit, testcases[0]) + f = file(os.path.join(datadir, '0000000001.out')) + o = string.join(f.readlines(), '\n') + f.close() + self.assertEqual(o, '3\0\n') + + self.st.cleanup(submit) + self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir)) + + def testBinary(self): + self.logger.info("TESTING: BINARY") + submit = self.ds.get_submit(6) + + # this program output '0xbb' + datadir = os.path.join(self.config.datadir, submit.id) + self.st.prepare(submit) + r = self.st.compile(submit) + self.assertTrue(r) + testcases = submit.get_testcases() + r = self.st.run(submit, testcases[0]) + f = file(os.path.join(datadir, '0000000001.out')) + o = string.join(f.readlines(), '\n') + f.close() + self.assertEqual(o, '\xbb') + + self.st.cleanup(submit) + self.assertFalse(not self.config.no_cleanup and os.path.exists(datadir)) + + def testOLE(self): + self.logger.info("TESTING: OLE") + submit = self.ds.get_submit(10) + + self.st.prepare(submit) + r = self.st.compile(submit) + self.assertTrue(r, "Submit compile failed") + ts = submit.get_testcases() + r = self.st.run(submit, ts[0]) + self.assertEqual(r[1], 'OLE') + + def testTLE(self): + self.logger.info("TESTING: TLE") + submit = self.ds.get_submit(11) + + self.st.prepare(submit) + self.st.compile(submit) + ts = submit.get_testcases() + r = self.st.run(submit, ts[0]) + self.assertEqual(r[1], 'TLE') + +class ComboTesterTestCase(OJTestCase): + + @classmethod + def suite(clazz): + tests = ( 'testGCC33', ) + return unittest.TestSuite(map(ComboTesterTestCase, tests)) + + def testGCC33(self): + self.ct = self.config.languages['gcc-3.3'] + submits = self.ds.get_submits(8) + r = self.ct.test(submits[2]) + self.assertTrue(r) + + from pysqlite2 import dbapi2 as sqlite + conn = sqlite.connect(self.dbname) + cur = conn.cursor() + cur.execute('DELETE FROM submit_test_result') + conn.commit() + cur.close() + conn.close() + +class LanguageTestCase(OJTestCase): + + @classmethod + def suite(clazz): + tests = ( 'testGCC33NOBC', 'testGCC33BC', 'testGXX33', + 'testJava5', 'testJava6', + 'testFreePascal20', + 'testPython25', + ) + return unittest.TestSuite(map(LanguageTestCase, tests)) + + def setUp(self): + OJTestCase.setUp(self) + + def testGCC33NOBC(self): + st = self.config.languages['gcc-3.3-nobc'] + submit = self.ds.get_submit(1001) + r = st.test(submit) + self.assertTrue(r, 'GCC NOBC test failed') + + def testGCC33BC(self): + st = self.config.languages['gcc-3.3-bc'] + submit = self.ds.get_submit(1001) + r = st.test(submit) + self.assertTrue(r, 'GCC WITH BC test failed') + + def testGXX33(self): + st = self.config.languages['g++-3.3'] + submit = self.ds.get_submit(1002) + r = st.test(submit) + self.assertTrue(r, 'GCC WITH BC test failed') + + def testJava5(self): + st = self.config.languages['java-1.6'] + submit = self.ds.get_submit(1004) + r = st.test(submit) + self.assertTrue(r, 'Java 5 test failed') + + def testJava6(self): + st = self.config.languages['java-1.6'] + submit = self.ds.get_submit(1004) + r = st.test(submit) + self.assertTrue(r, 'Java 6 test failed') + + def testFreePascal20(self): + st = self.config.languages['fpc-2.0'] + submit = self.ds.get_submit(1005) + r = st.test(submit) + self.assertTrue(r, 'FreePascal 2.0 test failed') + + def testPython25(self): + st = self.config.languages['python-2.5'] + submit = self.ds.get_submit(1006) + r = st.test(submit) + self.assertTrue(r, 'Python 2.5 test failed') + +class SecurityTestCase(OJTestCase): + """ + Security test including: + + * File Permission: only rundir are allowed to visit + * Forked Process: can oj kill all these process + * Daemon Process: can oj kill the daemon process + * Sleep: can oj kill sleep process + * Network: network usage should be forbidden + """ + + @classmethod + def suite(clazz): + tests = (#'testGCC33FilePermission', 'testJavaFilePermission', + #'testPython25FilePermission', + # 'testSleep', + 'testFork', 'testDaemon', + ) + return unittest.TestSuite(map(SecurityTestCase, tests)) + + def testGCC33FilePermission(self): + st = self.config.languages['gcc-3.3-nobc'] + submit = self.ds.get_submit(2311) + r = st.test(submit) + self.assertTrue(r, 'gcc 3.3 no bc security test failed') + + def testJavaFilePermission(self): + st = self.config.languages['java-1.6'] + submit = self.ds.get_submit(2331) + r = st.test(submit) + self.assertTrue(r, 'Java 1.6 security test failed') + + def testPython25FilePermission(self): + st = self.config.languages['python-2.5'] + submit = self.ds.get_submit(2361) + r = st.test(submit) + self.assertTrue(r, 'Python 2.5 security test failed') + + def testFork(self): + st = self.config.languages['gcc-3.3-nobc'] + submit = self.ds.get_submit(2412) + st.prepare(submit) + r = st.compile(submit) + self.assertTrue(r, "Failed") + ts = submit.get_testcases() + r = st.run(submit, ts[0]) + self.assertEqual(r[1], 'TLE', "Sleep test failed") + + def testDaemon(self): + st = self.config.languages['gcc-3.3-nobc'] + submit = self.ds.get_submit(2413) + st.prepare(submit) + r = st.compile(submit) + self.assertTrue(r, "Failed") + ts = submit.get_testcases() + r = st.run(submit, ts[0]) + self.assertEqual(r[1], 'TLE', "Sleep test failed") + + def testSleep(self): + st = self.config.languages['gcc-3.3-nobc'] + submit = self.ds.get_submit(2411) + st.prepare(submit) + r = st.compile(submit) + self.assertTrue(r, "Failed") + ts = submit.get_testcases() + r = st.run(submit, ts[0]) + self.assertEqual(r[1], 'TLE', "Sleep test failed") + + def testNetwork(self): + pass + +if __name__ == '__main__': + #unittest.main() + suite = unittest.TestSuite() + #suite.addTests(LanguageTestCase.suite()) + #suite.addTests(SimpleTesterTestCase.suite()) + #suite.addTest(ComboTesterTestCase.suite()) + suite.addTest(SecurityTestCase.suite()) + unittest.TextTestRunner().run(suite) + +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/python/tester.pyc b/python/tester.pyc new file mode 100644 index 0000000..012c2cd Binary files /dev/null and b/python/tester.pyc differ diff --git a/scripts/bash-guard b/scripts/bash-guard new file mode 100755 index 0000000..c6c678e --- /dev/null +++ b/scripts/bash-guard @@ -0,0 +1,7 @@ +#!/bin/dash + +# This is just a wrapper of run-guard program. +# AppArmor use different profile for different programs, so we create +# different scripts for different language. + +exec `dirname $0`/run-guard.py $@ diff --git a/scripts/binary-guard b/scripts/binary-guard new file mode 100755 index 0000000..c6c678e --- /dev/null +++ b/scripts/binary-guard @@ -0,0 +1,7 @@ +#!/bin/dash + +# This is just a wrapper of run-guard program. +# AppArmor use different profile for different programs, so we create +# different scripts for different language. + +exec `dirname $0`/run-guard.py $@ diff --git a/scripts/compare-file.py b/scripts/compare-file.py new file mode 100755 index 0000000..5830283 --- /dev/null +++ b/scripts/compare-file.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python + +import sys, string, hashlib, os +from stat import * + +def check_file(name, size, md5sum): + global result + + m = hashlib.md5() + if os.path.isfile(name): + s = os.stat(name)[ST_SIZE] + if s == string.atoi(size): + f = open(name, 'rb') + m.update(f.read(string.atoi(size))) + f.close() + d = m.hexdigest() + result.write("%s %d %s\n" % (name, s, d)) + return d == md5sum + else: + result.write("Size of %s is %d\n" % (name, s)) + else: + result.write("%s not found\n" % name) + return False + +if __name__ == '__main__': + result = open(sys.argv[3], 'w+') + fstdout = open(sys.argv[2], 'r') + for line in fstdout: + name, size, md5sum = string.split(string.strip(line), ' ') + if not check_file(name, size, md5sum): + print 'WA'; break + fstdout.close() + result.close() + print 'AC' diff --git a/scripts/compare-guard b/scripts/compare-guard new file mode 100755 index 0000000..e5ec1d7 --- /dev/null +++ b/scripts/compare-guard @@ -0,0 +1,10 @@ +#!/bin/dash + +scriptdir=`dirname $0` +prog=$scriptdir/compare-wrapper-$1 +if [ -e "$prog" ] ; then + shift + exec "$prog" "$@" +else + echo 'JSE' +fi diff --git a/scripts/compare-wrapper-bash-3 b/scripts/compare-wrapper-bash-3 new file mode 100755 index 0000000..2cfc6cf --- /dev/null +++ b/scripts/compare-wrapper-bash-3 @@ -0,0 +1,4 @@ +#!/bin/dash + +export PATH=$PATH:/bin:/usr/bin +exec /bin/bash "$@" diff --git a/scripts/compare-wrapper-fpc-2.2 b/scripts/compare-wrapper-fpc-2.2 new file mode 100755 index 0000000..8935622 --- /dev/null +++ b/scripts/compare-wrapper-fpc-2.2 @@ -0,0 +1,11 @@ +#!/bin/dash + +export PATH=$PATH:/bin:/usr/bin + +vdir=`dirname $1` +cdir=`pwd` +cd $vdir +/usr/bin/fpc -O3 -vw0 -dONLINE_JUDGE -Sd -omain -MDelphi `basename $1` >&2 +cd $cdir +shift +exec "$vdir/main" "$@" diff --git a/scripts/compare-wrapper-g++-3.3 b/scripts/compare-wrapper-g++-3.3 new file mode 100755 index 0000000..3364759 --- /dev/null +++ b/scripts/compare-wrapper-g++-3.3 @@ -0,0 +1,9 @@ +#!/bin/dash + +export PATH=$PATH:/bin:/usr/bin + +vdir=`dirname $1` +target=$vdir/main +/usr/bin/g++ -Wall -O2 -o $target $1 -lm +shift +exec "$vdir/main" "$@" diff --git a/scripts/compare-wrapper-gcc-3.3 b/scripts/compare-wrapper-gcc-3.3 new file mode 100755 index 0000000..c704926 --- /dev/null +++ b/scripts/compare-wrapper-gcc-3.3 @@ -0,0 +1,9 @@ +#!/bin/dash + +export PATH=$PATH:/bin:/usr/bin + +vdir=`dirname $1` +target=$vdir/main +/usr/bin/gcc -Wall -O2 -std=c99 -o $target $1 -lm >&2 +shift +exec "$vdir/main" "$@" diff --git a/scripts/compare-wrapper-gmcs-2.0 b/scripts/compare-wrapper-gmcs-2.0 new file mode 100755 index 0000000..c04acb3 --- /dev/null +++ b/scripts/compare-wrapper-gmcs-2.0 @@ -0,0 +1,11 @@ +#!/bin/dash + +export PATH=$PATH:/bin:/usr/bin + +vdir=`dirname $1` +cdir=`pwd` +cd $vdir +/usr/bin/gmcs "$1" +cd $cdir +shift +exec "$vdir/main.exe" "$@" diff --git a/scripts/compare-wrapper-java-1.5 b/scripts/compare-wrapper-java-1.5 new file mode 100755 index 0000000..ca31a27 --- /dev/null +++ b/scripts/compare-wrapper-java-1.5 @@ -0,0 +1,16 @@ +#!/bin/dash + +export PATH=$PATH:/bin:/usr/bin + +vdir=`dirname $1` +cdir=`pwd` +cd $vdir +JAVAC=javac +if [ -e '/usr/lib/jvm/java-6-sun/bin/javac ] ; then + JAVAC=/usr/lib/jvm/java-6-sun/bin/javac +else if [ -e '/usr/lib/jvm/java-6-openjdk/bin/javac ] ; then + JAVAC=/usr/lib/jvm/java-6-openjdk/bin/javac +fi +$JAVAC -source 1.5 -cp . "$@" +shift +exec java -cp "$vdir" Main "$@" diff --git a/scripts/compare-wrapper-java-1.6 b/scripts/compare-wrapper-java-1.6 new file mode 100755 index 0000000..ca31a27 --- /dev/null +++ b/scripts/compare-wrapper-java-1.6 @@ -0,0 +1,16 @@ +#!/bin/dash + +export PATH=$PATH:/bin:/usr/bin + +vdir=`dirname $1` +cdir=`pwd` +cd $vdir +JAVAC=javac +if [ -e '/usr/lib/jvm/java-6-sun/bin/javac ] ; then + JAVAC=/usr/lib/jvm/java-6-sun/bin/javac +else if [ -e '/usr/lib/jvm/java-6-openjdk/bin/javac ] ; then + JAVAC=/usr/lib/jvm/java-6-openjdk/bin/javac +fi +$JAVAC -source 1.5 -cp . "$@" +shift +exec java -cp "$vdir" Main "$@" diff --git a/scripts/compare-wrapper-python-2.5 b/scripts/compare-wrapper-python-2.5 new file mode 100755 index 0000000..57146f7 --- /dev/null +++ b/scripts/compare-wrapper-python-2.5 @@ -0,0 +1,8 @@ +#!/bin/dash + +export PATH=$PATH:/bin:/usr/bin +PYTHON=python +if [ -e /usr/bin/python2.6 ] ; then + PYTHON=python +fi +exec $PYTHON "$@" diff --git a/scripts/compare-wrapper-python-2.5old b/scripts/compare-wrapper-python-2.5old new file mode 100755 index 0000000..9e3b6f0 --- /dev/null +++ b/scripts/compare-wrapper-python-2.5old @@ -0,0 +1,8 @@ +#!/bin/dash + +export PATH=$PATH:/bin:/usr/bin +PYTHON=python3 +if [ -e /usr/bin/python2.6 ] ; then + PYTHON=python3 +fi +exec $PYTHON "$@" diff --git a/scripts/compile-guard b/scripts/compile-guard new file mode 100755 index 0000000..0462a78 --- /dev/null +++ b/scripts/compile-guard @@ -0,0 +1,6 @@ +#!/bin/dash + +ulimit -t 30 +cd $1 +shift 1 +exec $@ diff --git a/scripts/f b/scripts/f new file mode 100755 index 0000000..1abcf84 --- /dev/null +++ b/scripts/f @@ -0,0 +1,7 @@ +#!/bin/dash + +# This is just a wrapper of run-guard program. +# AppArmor use different profile for different programs, so we create +# different scripts for different language. +echo `dirname $0`/run-guard.py $@>fff.txt +exec `dirname $0`/run-guard.py $@ diff --git a/scripts/fpc b/scripts/fpc new file mode 100755 index 0000000..1cb675d --- /dev/null +++ b/scripts/fpc @@ -0,0 +1,4 @@ +#!/bin/dash + +export PATH=$PATH:/usr/bin:/bin +exec /usr/bin/fpc -O3 -vw0 -dONLINE_JUDGE -Sd -omain -MDelphi "$@" 2>&1 | sed '1d' | sed '1d' diff --git a/scripts/g++-3.3 b/scripts/g++-3.3 new file mode 100755 index 0000000..67b1b91 --- /dev/null +++ b/scripts/g++-3.3 @@ -0,0 +1,4 @@ +#!/bin/dash + +export PATH=$PATH:/usr/bin:/bin +exec /usr/bin/g++ -Wall -O2 -std=c++98 -DOJ -DONLINE_JUDGE -o main "$@" -lm diff --git a/scripts/gcc-3.3-bc b/scripts/gcc-3.3-bc new file mode 100755 index 0000000..c366046 --- /dev/null +++ b/scripts/gcc-3.3-bc @@ -0,0 +1,5 @@ +#!/bin/dash + +export PATH=$PATH:/usr/bin:/bin + +exec /usr/bin/gcc -Wall -O0 -g -std=c99 -DOJ -DONLINE_JUDGE -o main "$@" -lm diff --git a/scripts/gcc-3.3-nobc b/scripts/gcc-3.3-nobc new file mode 100755 index 0000000..f6b1962 --- /dev/null +++ b/scripts/gcc-3.3-nobc @@ -0,0 +1,4 @@ +#!/bin/dash + +export PATH=$PATH:/bin:/usr/bin +exec /usr/bin/gcc -Wall -O2 -std=c99 -DOJ -DONLINE_JUDGE -o main "$@" -lm diff --git a/scripts/gmcs-2.0 b/scripts/gmcs-2.0 new file mode 100755 index 0000000..f34dde0 --- /dev/null +++ b/scripts/gmcs-2.0 @@ -0,0 +1,4 @@ +#!/bin/dash + +export PATH=$PATH:/usr/bin:/bin +exec /usr/bin/gmcs "$@" diff --git a/scripts/java-guard b/scripts/java-guard new file mode 100755 index 0000000..c6c678e --- /dev/null +++ b/scripts/java-guard @@ -0,0 +1,7 @@ +#!/bin/dash + +# This is just a wrapper of run-guard program. +# AppArmor use different profile for different programs, so we create +# different scripts for different language. + +exec `dirname $0`/run-guard.py $@ diff --git a/scripts/javac-1.5 b/scripts/javac-1.5 new file mode 100755 index 0000000..38b9f3a --- /dev/null +++ b/scripts/javac-1.5 @@ -0,0 +1,4 @@ +#!/bin/dash + +export PATH=$PATH:/usr/bin:/bin +exec /usr/lib/jvm/java-6-sun/bin/javac -source 1.5 -cp . "$@" diff --git a/scripts/javac-1.6 b/scripts/javac-1.6 new file mode 100755 index 0000000..2592512 --- /dev/null +++ b/scripts/javac-1.6 @@ -0,0 +1,4 @@ +#!/bin/dash + +export PATH=$PATH:/usr/bin:/bin +exec /usr/lib/jvm/java-6-sun/bin/javac -source 1.6 -cp . "$@" diff --git a/scripts/mono-guard b/scripts/mono-guard new file mode 100755 index 0000000..c6c678e --- /dev/null +++ b/scripts/mono-guard @@ -0,0 +1,7 @@ +#!/bin/dash + +# This is just a wrapper of run-guard program. +# AppArmor use different profile for different programs, so we create +# different scripts for different language. + +exec `dirname $0`/run-guard.py $@ diff --git a/scripts/python-guard b/scripts/python-guard new file mode 100755 index 0000000..1abcf84 --- /dev/null +++ b/scripts/python-guard @@ -0,0 +1,7 @@ +#!/bin/dash + +# This is just a wrapper of run-guard program. +# AppArmor use different profile for different programs, so we create +# different scripts for different language. +echo `dirname $0`/run-guard.py $@>fff.txt +exec `dirname $0`/run-guard.py $@ diff --git a/scripts/python-guardd b/scripts/python-guardd new file mode 100755 index 0000000..499c445 --- /dev/null +++ b/scripts/python-guardd @@ -0,0 +1,6 @@ +#!/bin/dash + +# This is just a wrapper of run-guard program. +# AppArmor use different profile for different programs, so we create +# different scripts for different language. +exec `dirname $0`/run-guard.py $@ diff --git a/scripts/run-guard.py b/scripts/run-guard.py new file mode 100755 index 0000000..4f77dd5 --- /dev/null +++ b/scripts/run-guard.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python + +import os, sys, string, signal, resource, time, getopt, pickle + +class RunGuard: + + def __init__(self): + self.timelimit = 1 + self.memlimit = 65536 * 1024 + self.timetime = 5 + self.args = None + self.writeto = None + self.nproc = 1 + self.ofile = 32 + self.ldpreload = None + self.rundir = None + + self.usepickle = False + self.memrss = 0 + self.memdata = 0 + self.memstack = 0 + self.timeused = 0 + self.exitcode = 0 + self.sig = 0 + + def run(self): + self.parse_opts() + + self.childpid = os.fork() + if self.childpid == 0: + self.execute() + else: + self.monitor() + + self.write_result() + + def parse_opts(self): + try: + pos = sys.argv.index('-x') + optlist, self.args = getopt.gnu_getopt(sys.argv[:pos], 'e:t:m:d:o:T:p') + self.args = sys.argv[pos:] + except ValueError: + optlist, self.args = getopt.gnu_getopt(sys.argv, 'e:t:m:d:o:T:p') + for o, v in optlist: + if o == '-e': + self.nproc += string.atoi(v) + if o == '-t': + self.timelimit = string.atoi(v) + if o == '-m': + self.memlimit = string.atoi(v) * 1024 + if o == '-d': + self.rundir = v + if o == '-o': + self.writeto = v + if o == '-T': + self.timetime = string.atoi(v) + if o == '-p': + self.usepickle = True + + v = os.getenv('GUARD_RLIMIT_OFILE') + if v: self.ofile = string.atoi(v) + self.ldpreload = os.getenv('GUARD_LD_PRELOAD') + + def execute(self): + if self.rundir != None and os.path.isdir(self.rundir): + os.chdir(self.rundir) + + if self.nproc: + resource.setrlimit(resource.RLIMIT_NPROC, (self.nproc, self.nproc)) + if self.ofile: + resource.setrlimit(resource.RLIMIT_OFILE, (self.ofile, self.ofile)) + if self.ldpreload: + os.putenv('LD_PRELOAD', self.ldpreload) + + resource.setrlimit(resource.RLIMIT_CPU, (self.timelimit, self.timelimit)) + resource.setrlimit(resource.RLIMIT_AS, (self.memlimit, self.memlimit)) + os.execv(self.args[1], self.args[1:]) + sys.exit(127) # exit with JGE + + def monitor(self): + pid = 0 + remaintime = self.timelimit * self.timetime + while remaintime > 0: + pid, status, ru = os.wait4(self.childpid, os.WNOHANG) + self._get_memused() + if pid > 0: break + time.sleep(0.05) + remaintime -= 0.05 + + while pid == 0: + try: + os.kill(self.childpid, signal.SIGKILL) + except OSError, e: + pass + pid, status, ru = os.wait4(self.childpid, os.WNOHANG) + time.sleep(0.1) + + if os.WIFEXITED(status): + self.exitcode = os.WEXITSTATUS(status) + + if os.WIFSIGNALED(status): + self.sig = os.WTERMSIG(status) + + self.timeused = ru[0] + ru[1] + + def _get_memused(self): + procdir = '/proc/%d' % self.childpid + if os.path.isdir(procdir): + cmdline = file(procdir + '/cmdline', 'r').readlines() + + # do not get memory usage of this script after just fork + if len(cmdline) > 0 and \ + string.strip(cmdline[0], '\0') != \ + string.join(self.args[1:], '\0'): + return + + procstatus = file(procdir + '/status', 'r') + rss = 0; data = 0; stack = 0 + for line in procstatus: + n = line[0:6] + if n == 'VmRSS:': + rss = string.atoi(line[7:-3]) + if n == 'VmData': + data = string.atoi(line[8:-3]) + if n == 'VmStk:': + stack = string.atoi(line[7:-3]) + self.memrss = max(self.memrss, rss) + if self.memdata + self.memstack < data + stack: + self.memdata = data + self.memstack = stack + return + + def write_result(self): + if self.writeto == None: + f = sys.stdout + else: + f = file(self.writeto, 'w') + + if self.usepickle: + obj = { 'exitcode' : self.exitcode, + 'sig' : self.sig, + 'timeused' : self.timeused, + 'memrss' : self.memrss, + 'memdata' : self.memdata, + 'memstack' : self.memstack } + pickle.dump(obj, f) + else: + print >>f, "exitcode: %d" % self.exitcode + print >>f, "sig: %d" % self.sig + print >>f, "time: %.3f" % self.timeused + print >>f, "rss: %d" % self.memrss + print >>f, "data: %d" % self.memdata + print >>f, "stack: %d" % self.memstack + + if self.writeto != None: f.close() + +if __name__ == '__main__': + os.umask(0002) + RunGuard().run() diff --git a/utils/bitoj_adduser b/utils/bitoj_adduser new file mode 100755 index 0000000..9c83fb7 --- /dev/null +++ b/utils/bitoj_adduser @@ -0,0 +1,55 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +""" +这个程序用来向系统中添加帐号。这些帐号会在 bitoj 中使用，用户提交的程序会以 +不同的用户帐号执行。 + +""" + +import os, sys, string + +rundir = '/var/lib/bitoj' + +def usage(message = None): + if message: + print message + print "Usage: bitoj_adduser minnum maxnum" + +def adduser(usernum): + global rundir + + username = 'ojrun%02d' % usernum + homedir = os.path.join(rundir, username) + cmd = 'adduser --system --group --disabled-login --home %s %s' \ + % (homedir, username) + if os.system(cmd) == 0: + os.system('adduser oj %s' % username) + os.system('adduser %s oj' % username) + os.system('chmod g+w %s' % homedir) + os.system('setquota -u %s 25600 25600 1000 1000 /' % username) + +def main(): + if len(sys.argv) < 3: + usage() + sys.exit(1) + + minnum = string.atoi(sys.argv[1]) + maxnum = string.atoi(sys.argv[2]) + + if minnum > maxnum: + usage("minnum should be small than maxnum") + sys.exit(1) + + if minnum < 1 or maxnum > 100: + usage("minnum should between 1 and 100") + sys.exit(1) + + if maxnum < 1 or maxnum > 100: + usage("maxnum should between 1 and 100") + sys.exit(1) + + for i in range(minnum, maxnum + 1): + adduser(i) + +main() diff --git a/utils/xmlrpc-debug-proxy.py b/utils/xmlrpc-debug-proxy.py new file mode 100755 index 0000000..3d30839 --- /dev/null +++ b/utils/xmlrpc-debug-proxy.py @@ -0,0 +1,141 @@ +#!/usr/bin/python + +"""HTTP debugging proxy + +(Presumably) originally by Sam Rushing + http://www.nightmare.com/medusa/programming.html + +Modified by Phillip Pearson + http://www.myelin.co.nz/notes/xmlrpc-debug-proxy.html + (Changes placed in the public domain; do what you will) + + +A very small proxy for HTTP that dumps out what it sees, so you can debug your +XML-RPC without having to decipher the output from a packet sniffer. + +This is basically the proxy used in the Medusa asynchronous sockets tutorial +(available on http://www.nightmare.com/medusa/programming.html) with a minor +adjustment to make it flush its buffers before closing any connections. Without +that it will drop off important things like :) + +Syntax: xmlrpc-debug-proxy.py + +This will listen on port 8000+ and proxy through to : + +e.g. 'aproxy.py localhost 80' listens on localhost:8080 and proxies through to + the local web server on port 80. + +To debug stuff connecting to Radio, run 'xmlrpc-debug-proxy.py localhost 5335' +and point your scripts at http://localhost:13335/RPC2 (instead of +http://localhost:5335/RPC2) + +""" + +import asynchat +import asyncore +import socket +import string + +class proxy_server (asyncore.dispatcher): + + def __init__ (self, host, port): + asyncore.dispatcher.__init__ (self) + self.create_socket (socket.AF_INET, socket.SOCK_STREAM) + self.set_reuse_addr() + self.there = (host, port) + here = ('', port + 8000) + self.bind (here) + self.listen (5) + + def handle_accept (self): + print 'New connection' + proxy_receiver (self, self.accept()) + +class proxy_sender (asynchat.async_chat): + + "Sends data to the server" + + def __init__ (self, receiver, address): + asynchat.async_chat.__init__ (self) + self.receiver = receiver + self.set_terminator (None) + self.create_socket (socket.AF_INET, socket.SOCK_STREAM) + self.buffer = '' + self.set_terminator ('\n') + self.connect (address) + + def handle_connect (self): + print 'Sender connected' + + def collect_incoming_data (self, data): + self.buffer = self.buffer + data + + def found_terminator (self): + data = self.buffer + self.buffer = '' + print (u'==> (%d) %s' % (self.id, unicode(repr(data), 'utf-8'))).encode('utf-8') + self.receiver.push (data + '\n') + + def handle_close (self): + print 'Sender closing (inbuf len %d (%s), ac_in %d, ac_out %d )' % ( + len( self.buffer ), + self.buffer, + len( self.ac_in_buffer ), + len( self.ac_out_buffer ) + ) + + if len( self.buffer ): + self.found_terminator() + + self.receiver.close_when_done() + self.close() + +class proxy_receiver (asynchat.async_chat): + + "Receives data from the caller" + + channel_counter = 0 + + def __init__ (self, server, (conn, addr)): + asynchat.async_chat.__init__ (self, conn) + self.set_terminator ('\n') + self.server = server + self.id = self.channel_counter + self.channel_counter = self.channel_counter + 1 + self.sender = proxy_sender (self, server.there) + self.sender.id = self.id + self.buffer = '' + + def collect_incoming_data (self, data): + self.buffer = self.buffer + data + + def found_terminator (self): + import re + data = re.sub( r'\:8080', '', self.buffer ) + data = re.sub( r'localhost', self.server.there[0], data ) + self.buffer = '' + print (u'<== (%d) %s' % (self.id, unicode(repr(data), 'utf-8'))).encode('utf-8') + self.sender.push (data + '\n') + + def handle_close (self): + print 'Receiver closing (inbuf len %d (%s), ac_in %d, ac_out %d )' % ( + len( self.buffer ), + self.buffer, + len( self.ac_in_buffer ), + len( self.ac_out_buffer ) + ) + + if len( self.buffer ): + self.found_terminator() + + self.sender.close_when_done() + self.close() + +if __name__ == '__main__': + import sys + import string + if len(sys.argv) < 3: + print 'Usage: %s ' % sys.argv[0] + else: + ps = proxy_server (sys.argv[1], string.atoi (sys.argv[2])) + asyncore.loop()