From 603b230a8346c2445c7ba2f068fd1be04870b2e5 Mon Sep 17 00:00:00 2001 From: reginaldalfret Date: Mon, 19 Oct 2026 12:54:42 +0530 Subject: [PATCH 1/2] fix(contrib/sge): fix bytes pickle serialization in _dump and guard fstatvfs - In Python 3, pickle.dumps produces bytes. When running under __main__, rewriting module name with str raised TypeError, as did writing bytes to a file opened in text mode. Use protocol 0 with byte replacement and open with 'wb'. - Guard os.fstatvfs call against missing attribute or OSError on platforms such as Windows. Signed-off-by: reginaldalfret --- luigi/contrib/sge.py | 15 +++++++++------ test/contrib/sge_test.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/luigi/contrib/sge.py b/luigi/contrib/sge.py index 51dacd9003..d431eaa01b 100755 --- a/luigi/contrib/sge.py +++ b/luigi/contrib/sge.py @@ -218,8 +218,11 @@ def _init_local(self): random_id = "%016x" % random.getrandbits(64) folder_name = self.task_id + "-" + random_id self.tmp_dir = os.path.join(base_tmp_dir, folder_name) - max_filename_length = os.fstatvfs(0).f_namemax - self.tmp_dir = self.tmp_dir[:max_filename_length] + try: + max_filename_length = os.fstatvfs(0).f_namemax + self.tmp_dir = self.tmp_dir[:max_filename_length] + except (AttributeError, OSError): + pass logger.info("Tmp dir: %s", self.tmp_dir) os.makedirs(self.tmp_dir) @@ -258,10 +261,10 @@ def _dump(self, out_dir=""): with self.no_unpicklable_properties(): self.job_file = os.path.join(out_dir, "job-instance.pickle") if self.__module__ == "__main__": - d = pickle.dumps(self) - module_name = os.path.basename(sys.argv[0]).rsplit(".", 1)[0] - d = d.replace("(c__main__", "(c" + module_name) - with open(self.job_file, "w") as f: + d = pickle.dumps(self, protocol=0) + module_name = os.path.basename(sys.argv[0]).rsplit(".", 1)[0].encode("utf-8") + d = d.replace(b"(c__main__", b"(c" + module_name) + with open(self.job_file, "wb") as f: f.write(d) else: with open(self.job_file, "wb") as f: diff --git a/test/contrib/sge_test.py b/test/contrib/sge_test.py index 9a06350278..f69033e832 100644 --- a/test/contrib/sge_test.py +++ b/test/contrib/sge_test.py @@ -92,6 +92,21 @@ def test_run_job_with_dump(self, mock_check_output): luigi.build([task], local_scheduler=True) self.assertEqual(mock_check_output.call_count, 2) + def test_dump_main_module(self): + import tempfile + task = TestJobTask(i="2", n_cpu=1) + original_module = task.__module__ + try: + task.__module__ = "__main__" + with tempfile.TemporaryDirectory() as tmp_dir: + task._dump(tmp_dir) + self.assertTrue(os.path.exists(task.job_file)) + with open(task.job_file, "rb") as f: + content = f.read() + self.assertGreater(len(content), 0) + finally: + task.__module__ = original_module + def tearDown(self): for fpath in glob(os.path.join(DEFAULT_HOME, "test_file_*")): try: From 886939ff3a9ba956a26acd4f2dc0f02695cfb62a Mon Sep 17 00:00:00 2001 From: reginaldalfret Date: Mon, 19 Oct 2026 16:41:39 +0530 Subject: [PATCH 2/2] style(test): format test_dump_main_module with ruff Signed-off-by: reginaldalfret --- test/contrib/sge_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/contrib/sge_test.py b/test/contrib/sge_test.py index f69033e832..5cc9ab9bd8 100644 --- a/test/contrib/sge_test.py +++ b/test/contrib/sge_test.py @@ -94,6 +94,7 @@ def test_run_job_with_dump(self, mock_check_output): def test_dump_main_module(self): import tempfile + task = TestJobTask(i="2", n_cpu=1) original_module = task.__module__ try: