diff --git a/luigi/contrib/sge.py b/luigi/contrib/sge.py index 51dacd9003..d431eaa01b 100755 --- a/luigi/contrib/sge.py +++ b/luigi/contrib/sge.py @@ -218,8 +218,11 @@ def _init_local(self): random_id = "%016x" % random.getrandbits(64) folder_name = self.task_id + "-" + random_id self.tmp_dir = os.path.join(base_tmp_dir, folder_name) - max_filename_length = os.fstatvfs(0).f_namemax - self.tmp_dir = self.tmp_dir[:max_filename_length] + try: + max_filename_length = os.fstatvfs(0).f_namemax + self.tmp_dir = self.tmp_dir[:max_filename_length] + except (AttributeError, OSError): + pass logger.info("Tmp dir: %s", self.tmp_dir) os.makedirs(self.tmp_dir) @@ -258,10 +261,10 @@ def _dump(self, out_dir=""): with self.no_unpicklable_properties(): self.job_file = os.path.join(out_dir, "job-instance.pickle") if self.__module__ == "__main__": - d = pickle.dumps(self) - module_name = os.path.basename(sys.argv[0]).rsplit(".", 1)[0] - d = d.replace("(c__main__", "(c" + module_name) - with open(self.job_file, "w") as f: + d = pickle.dumps(self, protocol=0) + module_name = os.path.basename(sys.argv[0]).rsplit(".", 1)[0].encode("utf-8") + d = d.replace(b"(c__main__", b"(c" + module_name) + with open(self.job_file, "wb") as f: f.write(d) else: with open(self.job_file, "wb") as f: diff --git a/test/contrib/sge_test.py b/test/contrib/sge_test.py index 9a06350278..5cc9ab9bd8 100644 --- a/test/contrib/sge_test.py +++ b/test/contrib/sge_test.py @@ -92,6 +92,22 @@ def test_run_job_with_dump(self, mock_check_output): luigi.build([task], local_scheduler=True) self.assertEqual(mock_check_output.call_count, 2) + def test_dump_main_module(self): + import tempfile + + task = TestJobTask(i="2", n_cpu=1) + original_module = task.__module__ + try: + task.__module__ = "__main__" + with tempfile.TemporaryDirectory() as tmp_dir: + task._dump(tmp_dir) + self.assertTrue(os.path.exists(task.job_file)) + with open(task.job_file, "rb") as f: + content = f.read() + self.assertGreater(len(content), 0) + finally: + task.__module__ = original_module + def tearDown(self): for fpath in glob(os.path.join(DEFAULT_HOME, "test_file_*")): try: