Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Framework/Core/include/Framework/InputSpec.h
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ struct InputSpec {

enum Lifetime lifetime;

/// Whether the input is enabled. Disabled inputs are optional inputs
/// which are not going to be used.
bool enabled = true;

/// A set of configurables which can be used to customise the InputSpec.
std::vector<ConfigParamSpec> metadata;

Expand Down
3 changes: 3 additions & 0 deletions Framework/Core/include/Framework/OutputSpec.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ struct OutputSpec {
std::variant<ConcreteDataMatcher, ConcreteDataTypeMatcher> matcher;
#endif
enum Lifetime lifetime = Lifetime::Timeframe;
/// Whether the output is enabled. Disabled outputs are optional outputs
/// which are not going to be produced.
bool enabled = true;

/// Build a fully qualified tuple for the OutputSpec
OutputSpec(OutputLabel const& inBinding, header::DataOrigin inOrigin, header::DataDescription inDescription,
Expand Down
35 changes: 35 additions & 0 deletions Framework/Core/src/WorkflowSerializationHelpers.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
IN_INPUT_DESCRIPTION_REF,
IN_INPUT_SUBSPEC_REF,
IN_INPUT_LIFETIME,
IN_INPUT_ENABLED,
IN_INPUT_STARTTIME,
IN_INPUT_MATCHER,
IN_INPUT_MATCHER_OPERATION,
Expand All @@ -78,6 +79,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
IN_OUTPUT_DESCRIPTION,
IN_OUTPUT_SUBSPEC,
IN_OUTPUT_LIFETIME,
IN_OUTPUT_ENABLED,
IN_OUTPUT_OPTIONS,
IN_OPTION,
IN_OPTION_NAME,
Expand Down Expand Up @@ -192,6 +194,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
case State::IN_INPUT_LIFETIME:
s << "IN_INPUT_LIFETIME";
break;
case State::IN_INPUT_ENABLED:
s << "IN_INPUT_ENABLED";
break;
case State::IN_INPUT_STARTTIME:
s << "IN_INPUT_STARTTIME";
break;
Expand All @@ -216,6 +221,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
case State::IN_OUTPUT_LIFETIME:
s << "IN_OUTPUT_LIFETIME";
break;
case State::IN_OUTPUT_ENABLED:
s << "IN_OUTPUT_ENABLED";
break;
case State::IN_OUTPUT_OPTIONS:
s << "IN_OUTPUT_OPTIONS";
break;
Expand Down Expand Up @@ -371,6 +379,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
} else {
dataProcessors.back().inputs.push_back(InputSpec({binding}, std::move(*matcher), lifetime, inputOptions));
}
dataProcessors.back().inputs.back().enabled = enabled;
// "enabled" is optional in the JSON, so reset to the default
enabled = true;
inputMatcherNodes.clear();
inputOptions.clear();

Expand Down Expand Up @@ -434,6 +445,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
} else {
dataProcessors.back().outputs.push_back(OutputSpec({binding}, {origin, description}, lifetime, outputOptions));
}
dataProcessors.back().outputs.back().enabled = enabled;
// "enabled" is optional in the JSON, so reset to the default
enabled = true;
outputOptions.clear();
outputHasSubSpec = false;
} else if (in(State::IN_OPTION)) {
Expand Down Expand Up @@ -649,6 +663,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
push(State::IN_INPUT_STARTTIME);
} else if (in(State::IN_INPUT) && strncmp(str, "lifetime", length) == 0) {
push(State::IN_INPUT_LIFETIME);
} else if (in(State::IN_INPUT) && strncmp(str, "enabled", length) == 0) {
push(State::IN_INPUT_ENABLED);
} else if (in(State::IN_INPUT) && strncmp(str, "starttime", length) == 0) {
push(State::IN_INPUT_STARTTIME);
} else if (in(State::IN_INPUT) && strncmp(str, "metadata", length) == 0) {
Expand All @@ -664,6 +680,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
outputHasSubSpec = true;
} else if (in(State::IN_OUTPUT) && strncmp(str, "lifetime", length) == 0) {
push(State::IN_OUTPUT_LIFETIME);
} else if (in(State::IN_OUTPUT) && strncmp(str, "enabled", length) == 0) {
push(State::IN_OUTPUT_ENABLED);
} else if (in(State::IN_OUTPUT) && strncmp(str, "metadata", length) == 0) {
push(State::IN_OUTPUT_OPTIONS);
} else if (in(State::IN_DATAPROCESSOR) && strncmp(str, "name", length) == 0) {
Expand Down Expand Up @@ -848,6 +866,18 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
return true;
}

bool Bool(bool b)
{
O2_SIGNPOST_EVENT_EMIT(workflow_importer, _o2_signpost_id_t{(int64_t)states.size()}, "import", "Bool(%s)", b ? "true" : "false");
if (in(State::IN_INPUT_ENABLED)) {
enabled = b;
} else if (in(State::IN_OUTPUT_ENABLED)) {
enabled = b;
}
pop();
return true;
}

bool Int(int i)
{
O2_SIGNPOST_EVENT_EMIT(workflow_importer, _o2_signpost_id_t{(int64_t)states.size()}, "import", "Int(%d)", i);
Expand Down Expand Up @@ -918,6 +948,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
size_t subspec;
size_t ref;
Lifetime lifetime;
bool enabled = true;
std::string metadatumKey;
std::string metadatumValue;
std::string optionName;
Expand Down Expand Up @@ -1117,6 +1148,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
}
w.Key("lifetime");
w.Uint((int)input.lifetime);
w.Key("enabled");
w.Bool(input.enabled);
if (input.metadata.empty() == false) {
w.Key("metadata");
w.StartArray();
Expand Down Expand Up @@ -1166,6 +1199,8 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
}
w.Key("lifetime");
w.Uint((int)output.lifetime);
w.Key("enabled");
w.Bool(output.enabled);
if (output.metadata.empty() == false) {
w.Key("metadata");
w.StartArray();
Expand Down
48 changes: 32 additions & 16 deletions Framework/Core/test/test_FrameworkDataFlowToDDS.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -169,14 +169,16 @@ TEST_CASE("TestDDS")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;A1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
},
{
&quot;binding&quot;: &quot;TST/A2/0&quot;,
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;A2&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;options&quot;: [],
Expand All @@ -195,7 +197,8 @@ TEST_CASE("TestDDS")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;A1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;outputs&quot;: [
Expand All @@ -204,7 +207,8 @@ TEST_CASE("TestDDS")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;B1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;options&quot;: [],
Expand All @@ -223,7 +227,8 @@ TEST_CASE("TestDDS")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;A2&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;outputs&quot;: [
Expand All @@ -232,7 +237,8 @@ TEST_CASE("TestDDS")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;C1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;options&quot;: [],
Expand All @@ -251,14 +257,16 @@ TEST_CASE("TestDDS")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;B1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
},
{
&quot;binding&quot;: &quot;y&quot;,
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;C1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;outputs&quot;: [],
Expand Down Expand Up @@ -434,14 +442,16 @@ TEST_CASE("TestDDSExpendable")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;A1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
},
{
&quot;binding&quot;: &quot;TST/A2/0&quot;,
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;A2&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;options&quot;: [],
Expand All @@ -460,7 +470,8 @@ TEST_CASE("TestDDSExpendable")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;A1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;outputs&quot;: [
Expand All @@ -469,7 +480,8 @@ TEST_CASE("TestDDSExpendable")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;B1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;options&quot;: [],
Expand All @@ -488,7 +500,8 @@ TEST_CASE("TestDDSExpendable")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;A2&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;outputs&quot;: [
Expand All @@ -497,7 +510,8 @@ TEST_CASE("TestDDSExpendable")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;C1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;options&quot;: [],
Expand All @@ -516,14 +530,16 @@ TEST_CASE("TestDDSExpendable")
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;B1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
},
{
&quot;binding&quot;: &quot;y&quot;,
&quot;origin&quot;: &quot;TST&quot;,
&quot;description&quot;: &quot;C1&quot;,
&quot;subspec&quot;: 0,
&quot;lifetime&quot;: 0
&quot;lifetime&quot;: 0,
&quot;enabled&quot;: true
}
],
&quot;outputs&quot;: [],
Expand Down
71 changes: 71 additions & 0 deletions Framework/Core/test/test_WorkflowSerialization.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include "Framework/WorkflowSpec.h"
#include "../src/WorkflowSerializationHelpers.h"
#include <catch_amalgamated.hpp>
#include <regex>

using namespace o2::framework;

Expand Down Expand Up @@ -169,3 +170,73 @@ TEST_CASE("TestInputOutputSpecMetadata")
REQUIRE(wso[0].outputs[0].metadata[0] == wsi[0].outputs[0].metadata[0]);
REQUIRE(wso[0].outputs[0].metadata[1] == wsi[0].outputs[0].metadata[1]);
}

TEST_CASE("TestInputOutputSpecEnabled")
{
InputSpec i1{"i1", "TST", "INPUT1", 0};
InputSpec i2{"i2", "TST", "INPUT2", 1};
i2.enabled = false;
// wildcard on subspec, imported via the DataDescriptorMatcher path
InputSpec i3{"i3", ConcreteDataTypeMatcher{"TST", "INPUT3"}};
i3.enabled = false;
OutputSpec o1{OutputLabel{"o1"}, "TST", "OUTPUT1", 0};
OutputSpec o2{OutputLabel{"o2"}, ConcreteDataTypeMatcher{"TST", "OUTPUT2"}};
o2.enabled = false;
OutputSpec o3{OutputLabel{"o3"}, "TST", "OUTPUT3", 3};
o3.enabled = false;

WorkflowSpec wso{
DataProcessorSpec{
.name = "S1",
.inputs = {i1, i2, i3},
.outputs = {o1, o2, o3}}};

std::vector<DataProcessorInfo> dataProcessorInfoOut{
{.name = "S1", .executable = "test_Framework_test_SerializationWorkflow"},
};
CommandInfo commandInfoOut{"o2-dpl-workflow -b"};

std::ostringstream firstDump;
WorkflowSerializationHelpers::dump(firstDump, wso, dataProcessorInfoOut, commandInfoOut);

std::vector<DataProcessorInfo> dataProcessorInfoIn{};
CommandInfo commandInfoIn;
std::istringstream is;
is.str(firstDump.str());
WorkflowSpec wsi;
REQUIRE(WorkflowSerializationHelpers::import(is, wsi, dataProcessorInfoIn, commandInfoIn));

REQUIRE(wsi.size() == 1);
REQUIRE(wsi[0].inputs.size() == wso[0].inputs.size());
REQUIRE(wsi[0].outputs.size() == wso[0].outputs.size());
for (size_t i = 0; i < wso[0].inputs.size(); ++i) {
REQUIRE(wsi[0].inputs[i].enabled == wso[0].inputs[i].enabled);
}
for (size_t i = 0; i < wso[0].outputs.size(); ++i) {
REQUIRE(wsi[0].outputs[i].enabled == wso[0].outputs[i].enabled);
}

std::ostringstream secondDump;
WorkflowSerializationHelpers::dump(secondDump, wsi, dataProcessorInfoIn, commandInfoIn);
REQUIRE(firstDump.str() == secondDump.str());

// Backward compatibility: a JSON without the "enabled" key
// must import everything as enabled.
auto legacy = std::regex_replace(firstDump.str(), std::regex(R"(,\s*"enabled":\s*(true|false))"), "");
REQUIRE(legacy.find("\"enabled\"") == std::string::npos);
std::istringstream lis;
lis.str(legacy);
WorkflowSpec wsl;
std::vector<DataProcessorInfo> dataProcessorInfoLegacy{};
CommandInfo commandInfoLegacy;
REQUIRE(WorkflowSerializationHelpers::import(lis, wsl, dataProcessorInfoLegacy, commandInfoLegacy));
REQUIRE(wsl.size() == 1);
REQUIRE(wsl[0].inputs.size() == wso[0].inputs.size());
REQUIRE(wsl[0].outputs.size() == wso[0].outputs.size());
for (auto const& input : wsl[0].inputs) {
REQUIRE(input.enabled);
}
for (auto const& output : wsl[0].outputs) {
REQUIRE(output.enabled);
}
}