Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion luigi/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -877,6 +877,8 @@ def add_task(
task.param_visibilities = _get_default(param_visibilities, {})
if not task.params:
task.set_params(params)
if "cmdline_params" in kwargs:
task.cmdline_params = kwargs["cmdline_params"]

if batch_id is not None:
task.batch_id = batch_id
Expand Down Expand Up @@ -1575,14 +1577,17 @@ def re_enable_task(self, task_id):
def fetch_error(self, task_id, **kwargs):
if self._state.has_task(task_id):
task = self._state.get_task(task_id)
return {
response = {
"taskId": task_id,
"error": task.expl,
"displayName": task.pretty_id,
"taskParams": task.params,
"taskModule": task.module,
"taskFamily": task.family,
}
if hasattr(task, "cmdline_params"):
response["taskCmdline"] = task.cmdline_params
return response
else:
return {"taskId": task_id, "error": ""}

Expand Down
5 changes: 5 additions & 0 deletions luigi/static/visualiser/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,13 @@ <h4 class="modal-title" id="myModalLabel">Traceback for {{displayName}}</h4>
</div>
<div class="modal-body">
<pre class="pre-scrollable">{{error}}</pre>
{{#rerunUnavailable}}
A re-run command is unavailable for these boolean or optional parameter values.
{{/rerunUnavailable}}
{{^rerunUnavailable}}
Command to re-run:
<pre class="pre-scrollable">luigi --module {{taskModule}} {{taskFamily}} {{taskParams}}</pre>
{{/rerunUnavailable}}
</div>
<div class="modal-footer">
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
Expand Down
5 changes: 4 additions & 1 deletion luigi/static/visualiser/js/visualiserApp.js
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,10 @@ function visualiserApp(luigi) {

function showErrorTrace(data) {
data.error = decodeError(data.error);
if (data.taskParams) {
if ('taskCmdline' in data) {
data.rerunUnavailable = data.taskCmdline === null;
data.taskParams = data.taskCmdline;
} else if (data.taskParams) {
data.taskParams = Object.entries(data.taskParams).map(([k,v]) => `--${k.replace(/_/g, '-')} ${JSON.stringify(v)}`).join(" ");
}
$("#errorModal").empty().append(renderTemplate("errorTemplate", data));
Expand Down
26 changes: 26 additions & 0 deletions luigi/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import json
import logging
import re
import shlex
import traceback
import warnings
from collections import OrderedDict, deque
Expand Down Expand Up @@ -539,6 +540,31 @@ def to_str_params(self, only_significant=False, only_public=False):

return params_str

def _get_cmdline_params(self):
"""Format rerun parameters for a POSIX shell, or None if omission cannot reproduce a value."""
args = []
serialized = self.to_str_params()
for name, param in self.get_params():
if name not in serialized:
continue
value = self.param_kwargs[name]
flag = "--" + name.replace("_", "-")
implicit_bool = isinstance(param, parameter.BoolParameter) and param.parsing == param.IMPLICIT_PARSING
if (isinstance(param, parameter.OptionalParameterMixin) and value is None) or (implicit_bool and not value):
try:
if param.task_value(self.task_family, name) is not value:
return None
except Exception:
# A valid explicit value may override an unusable configured default.
return None
continue
if implicit_bool:
args.append(flag)
else:
serialized_value = str(value).lower() if isinstance(param, parameter.BoolParameter) else serialized[name]
args.append(flag + "=" + serialized_value)
return shlex.join(args)

def _get_param_visibilities(self):
param_visibilities = {}
params = dict(self.get_params())
Expand Down
1 change: 1 addition & 0 deletions luigi/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,7 @@ def _add_task(self, *args, **kwargs):

if task and kwargs.get("params"):
kwargs["param_visibilities"] = task._get_param_visibilities()
kwargs["cmdline_params"] = task._get_cmdline_params()

self._scheduler.add_task(*args, **kwargs)

Expand Down
149 changes: 149 additions & 0 deletions test/rerun_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import pickle
import shlex

from helpers import LuigiTestCase, with_config

import luigi
from luigi.cmdline_parser import CmdlineParser
from luigi.scheduler import Scheduler
from luigi.worker import Worker


class RerunTest(LuigiTestCase):
def assert_round_trip(self, task):
command = task._get_cmdline_params()
self.assertIsNotNone(command)
with CmdlineParser.global_instance([task.task_family, *shlex.split(command)]) as parser:
restored = parser.get_task_obj()
self.assertEqual(restored.param_kwargs, task.param_kwargs)
self.assertEqual(restored.task_id, task.task_id)
return command

def test_implicit_bool(self):
class T(luigi.Task):
rerun = luigi.BoolParameter()

self.assertEqual(self.assert_round_trip(T(rerun=False)), "")
self.assertEqual(self.assert_round_trip(T(rerun=True)), "--rerun")
self.assertEqual(T(rerun=False).to_str_params(), {"rerun": "False"})

def test_explicit_bool(self):
class T(luigi.Task):
rerun = luigi.BoolParameter(default=True, parsing=luigi.BoolParameter.EXPLICIT_PARSING)

for value in (True, False):
with self.subTest(value=value):
self.assertEqual(self.assert_round_trip(T(rerun=value)), "--rerun=" + str(value).lower())

def test_optional_and_empty_string(self):
class T(luigi.Task):
absent = luigi.OptionalParameter()
count = luigi.OptionalIntParameter()
empty = luigi.Parameter(default="")
text = luigi.Parameter()

for text in ("False", "null", "two words", 'quoted "value"', "it's $HOME `literal`", "-foo", "--other"):
with self.subTest(text=text):
command = self.assert_round_trip(T(text=text))
self.assertNotIn("--absent", command)
self.assertNotIn("--count", command)
self.assertIn("--empty=", command)
self.assert_round_trip(T(text="value", absent="present", count=0))

def test_optional_explicit_bool(self):
class T(luigi.Task):
enabled = luigi.OptionalBoolParameter(parsing=luigi.BoolParameter.EXPLICIT_PARSING)

for value in (None, False, True):
with self.subTest(value=value):
self.assert_round_trip(T(enabled=value))

def test_private_parameter_not_in_command(self):
class T(luigi.Task):
secret = luigi.Parameter(default="do-not-show", visibility=luigi.parameter.ParameterVisibility.PRIVATE)
enabled = luigi.BoolParameter()

self.assertEqual(T(enabled=True)._get_cmdline_params(), "--enabled")

def test_unrepresentable_implicit_false(self):
class T(luigi.Task):
enabled = luigi.BoolParameter(default=True)

self.assertIsNone(T(enabled=False)._get_cmdline_params())
self.assert_round_trip(T(enabled=True))

@with_config({"T": {"enabled": "true"}})
def test_unrepresentable_configured_implicit_false(self):
class T(luigi.Task):
enabled = luigi.BoolParameter()

self.assertIsNone(T(enabled=False)._get_cmdline_params())

def test_unrepresentable_optional_none(self):
class T(luigi.Task):
value = luigi.OptionalParameter(default="configured")

self.assertIsNone(T(value=None)._get_cmdline_params())

@with_config({"T": {"enabled": "not-a-bool"}})
def test_explicit_value_overrides_invalid_config(self):
class T(luigi.Task):
enabled = luigi.BoolParameter()

def complete(self):
return False

task = T(enabled=False)
scheduler = Scheduler()
with Worker(scheduler=scheduler, no_install_shutdown_handler=True) as worker:
self.assertTrue(worker.add(task))
self.assertIsNone(scheduler.fetch_error(task.task_id)["taskCmdline"])

@with_config({"T": {"count": "not-an-int"}})
def test_optional_none_overrides_invalid_config(self):
class T(luigi.Task):
count = luigi.OptionalIntParameter()

self.assertIsNone(T(count=None)._get_cmdline_params())

def test_worker_scheduler_command(self):
class T(luigi.Task):
enabled = luigi.BoolParameter()
absent = luigi.OptionalParameter()

def complete(self):
return False

scheduler = Scheduler(prune_on_get_work=False)
task = T(enabled=True)
with Worker(scheduler=scheduler, no_install_shutdown_handler=True) as worker:
worker.add(task)
# JSON serialization and persisted scheduler state both preserve the command.
scheduler._state._tasks = pickle.loads(pickle.dumps(scheduler._state._tasks))
response = json.loads(json.dumps(scheduler.fetch_error(task.task_id)))
self.assertEqual(response["taskCmdline"], "--enabled")
self.assertEqual(response["taskParams"], {"enabled": "True", "absent": ""})
scheduler.add_task(worker="older-worker", task_id=task.task_id, status="FAILED")
self.assertEqual(scheduler.fetch_error(task.task_id)["taskCmdline"], "--enabled")

def test_scheduler_legacy_and_unavailable(self):
scheduler = Scheduler()
scheduler.add_task(worker="worker", task_id="legacy", params={"flag": "False"})
self.assertNotIn("taskCmdline", scheduler.fetch_error("legacy"))
scheduler.add_task(worker="worker", task_id="unavailable", params={"flag": "False"}, cmdline_params=None)
self.assertIsNone(scheduler.fetch_error("unavailable")["taskCmdline"])
scheduler.add_task(worker="worker", task_id="empty", params={"flag": "False"}, cmdline_params="")
self.assertEqual(scheduler.fetch_error("empty")["taskCmdline"], "")
Loading