diff --git a/SEFramework/SEFramework/Output/OutputRegistry.h b/SEFramework/SEFramework/Output/OutputRegistry.h index 83c5b0ae0..48ccdf53a 100644 --- a/SEFramework/SEFramework/Output/OutputRegistry.h +++ b/SEFramework/SEFramework/Output/OutputRegistry.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -35,18 +36,40 @@ namespace SourceXtractor { class OutputRegistry { - public: - template using ColumnConverter = std::function; using SourceToRowConverter = std::function; + class ColumnFromSource { + public: + template + explicit ColumnFromSource(ColumnConverter converter) { + m_convert_func = [converter](const SourceInterface& source, std::size_t i) { + return converter(source.getProperty(i)); + }; + } + Euclid::Table::Row::cell_type operator()(const SourceInterface& source) const { + return m_convert_func(source, index); + } + std::size_t index = 0; + + private: + std::function m_convert_func; + }; + + struct ColInfo { + std::string unit; + std::string description; + }; + +public: template void registerColumnConverter(std::string column_name, ColumnConverter converter, std::string column_unit="", std::string column_description="") { m_property_to_names_map[typeid(PropertyType)].emplace_back(column_name); + m_name_to_property_map.emplace(column_name, typeid(PropertyType)); std::type_index conv_out_type = typeid(OutType); ColumnFromSource conv_func {converter}; m_name_to_converter_map.emplace(column_name, @@ -54,6 +77,14 @@ class OutputRegistry { m_name_to_col_info_map.emplace(column_name, ColInfo{column_unit, column_description}); } + std::type_index getPropertyForColumn(const std::string& column_name) const { + return m_name_to_property_map.at(column_name); + } + + const std::pair& getColumnConverter(const std::string& column_name) const { + return m_name_to_converter_map.at(column_name); + } + /** * When there are multiple instances of a given property, generate one column output with the given suffix for each * instance @@ -147,7 +178,7 @@ class OutputRegistry { m_output_properties.emplace(alias_name, typeid(PropertyType)); } - std::set getOutputPropertyNames() { + std::set getOutputPropertyNames() const { std::set result {}; for (auto& pair : m_output_properties) { result.emplace(pair.first); @@ -155,38 +186,20 @@ class OutputRegistry { return result; } + std::set getColumns(const std::string& property) const; + + std::set getColumns(const PropertyId& property) const; + SourceToRowConverter getSourceToRowConverter(const std::vector& enabled_optional); void printPropertyColumnMap(const std::vector& properties={}); private: - - class ColumnFromSource { - public: - template - explicit ColumnFromSource(ColumnConverter converter) { - m_convert_func = [converter](const SourceInterface& source, std::size_t i){ - return converter(source.getProperty(i)); - }; - } - Euclid::Table::Row::cell_type operator()(const SourceInterface& source) { - return m_convert_func(source, index); - } - std::size_t index = 0; - private: - std::function m_convert_func; - }; - - struct ColInfo { - std::string unit; - std::string description; - }; - - std::map> m_property_to_names_map {}; - std::map> m_name_to_converter_map {}; - std::map m_name_to_col_info_map {}; - std::multimap m_output_properties {}; - + std::map> m_property_to_names_map{}; + std::map m_name_to_property_map{}; + std::map> m_name_to_converter_map{}; + std::map m_name_to_col_info_map{}; + std::multimap m_output_properties{}; }; } /* namespace SourceXtractor */ diff --git a/SEFramework/SEFramework/Pipeline/Measurement.h b/SEFramework/SEFramework/Pipeline/Measurement.h index ece9264e3..60af09e58 100644 --- a/SEFramework/SEFramework/Pipeline/Measurement.h +++ b/SEFramework/SEFramework/Pipeline/Measurement.h @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -39,6 +40,7 @@ class Measurement : public PipelineReceiver, public Pipeli virtual void startThreads() = 0; virtual void stopThreads() = 0; virtual void synchronizeThreads() = 0; + virtual void cancel() = 0; }; } diff --git a/SEFramework/SEFramework/Property/PropertyHolder.h b/SEFramework/SEFramework/Property/PropertyHolder.h index de95a8288..133bc882e 100644 --- a/SEFramework/SEFramework/Property/PropertyHolder.h +++ b/SEFramework/SEFramework/Property/PropertyHolder.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -44,6 +45,8 @@ class PropertyHolder { public: + using const_iterator = std::unordered_map>::const_iterator; + /// Destructor virtual ~PropertyHolder() = default; @@ -60,16 +63,26 @@ class PropertyHolder { const Property& getProperty(const PropertyId& property_id) const; /// Sets a property, overwriting it if necessary - void setProperty(std::unique_ptr property, const PropertyId& property_id); + void setProperty(std::shared_ptr property, const PropertyId& property_id); /// Returns true if the property is set bool isPropertySet(const PropertyId& property_id) const; - + void clear(); + void update(const PropertyHolder& other); + + const_iterator begin() const { + return m_properties.begin(); + } + + const_iterator end() const { + return m_properties.end(); + } + private: - std::unordered_map> m_properties; + std::unordered_map> m_properties; }; /* End of ObjectWithProperties class */ diff --git a/SEFramework/SEFramework/Property/PropertyId.h b/SEFramework/SEFramework/Property/PropertyId.h index dae899553..aa1b96e52 100644 --- a/SEFramework/SEFramework/Property/PropertyId.h +++ b/SEFramework/SEFramework/Property/PropertyId.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/* + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -39,6 +40,8 @@ namespace SourceXtractor { class PropertyId { public: + PropertyId(std::type_index type_id, unsigned int index) : m_type_id(type_id), m_index(index) {} + /// Templated factory method used to create a PropertyId based on the type of a property. /// An optional index parameter is used to make the distinction between several properties of the same type. template @@ -74,12 +77,9 @@ class PropertyId { std::string getString() const; private: - PropertyId(std::type_index type_id, unsigned int index) : m_type_id(type_id), m_index(index) {} std::type_index m_type_id; unsigned int m_index; - - friend struct std::hash; }; diff --git a/SEFramework/SEFramework/Source/SimpleSource.h b/SEFramework/SEFramework/Source/SimpleSource.h index 97e3d09be..5becd205f 100644 --- a/SEFramework/SEFramework/Source/SimpleSource.h +++ b/SEFramework/SEFramework/Source/SimpleSource.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -24,8 +25,9 @@ #ifndef _SEFRAMEWORK_SOURCE_SIMPLESOURCE_H_ #define _SEFRAMEWORK_SOURCE_SIMPLESOURCE_H_ -#include "SEFramework/Source/SourceInterface.h" +#include "AlexandriaKernel/memory_tools.h" #include "SEFramework/Property/PropertyHolder.h" +#include "SEFramework/Source/SourceInterface.h" namespace SourceXtractor { @@ -54,6 +56,18 @@ class SimpleSource : public SourceInterface { /// Constructor SimpleSource() {} + std::unique_ptr clone() const override { + auto cloned = Euclid::make_unique(); + cloned->m_property_holder.update(m_property_holder); + return std::move(cloned); + } + + void visitProperties(const PropertyVisitor& visitor) override { + std::for_each( + m_property_holder.begin(), m_property_holder.end(), + [visitor](const std::pair>& prop) { visitor(prop.first, prop.second); }); + } + // Note : Because the get/setProperty() methods of the SourceInterface are // templated, the overrides of the non-templated versions will hide them. For // this reason it is necessary to re-introduce the templated methods, which is @@ -68,7 +82,7 @@ class SimpleSource : public SourceInterface { return m_property_holder.getProperty(property_id); } - void setProperty(std::unique_ptr property, const PropertyId& property_id) override { + void setProperty(std::shared_ptr property, const PropertyId& property_id) override { m_property_holder.setProperty(std::move(property), property_id); } diff --git a/SEFramework/SEFramework/Source/SimpleSourceGroup.h b/SEFramework/SEFramework/Source/SimpleSourceGroup.h index fc9130eb4..2b31fe8cf 100644 --- a/SEFramework/SEFramework/Source/SimpleSourceGroup.h +++ b/SEFramework/SEFramework/Source/SimpleSourceGroup.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -62,6 +63,10 @@ class SimpleSourceGroup : public SourceGroupInterface { void merge(SourceGroupInterface&& other) override; + std::unique_ptr clone() const override; + + void visitProperties(const PropertyVisitor& visitor) override; + using SourceInterface::getProperty; using SourceInterface::setProperty; @@ -69,7 +74,7 @@ class SimpleSourceGroup : public SourceGroupInterface { const Property& getProperty(const PropertyId& property_id) const override; - void setProperty(std::unique_ptr property, const PropertyId& property_id) override; + void setProperty(std::shared_ptr property, const PropertyId& property_id) override; private: diff --git a/SEFramework/SEFramework/Source/SourceGroupInterface.h b/SEFramework/SEFramework/Source/SourceGroupInterface.h index 901698a17..e527b0b3a 100644 --- a/SEFramework/SEFramework/Source/SourceGroupInterface.h +++ b/SEFramework/SEFramework/Source/SourceGroupInterface.h @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -23,6 +24,7 @@ #define _SEFRAMEWORK_SOURCEGROUPINTERFACE_H #include "SEFramework/Source/SourceInterface.h" +#include "AlexandriaKernel/memory_tools.h" #include namespace SourceXtractor { @@ -35,7 +37,7 @@ namespace SourceXtractor { * */ -class SourceGroupInterface : protected SourceInterface { +class SourceGroupInterface : public SourceInterface { template using CollectionType = typename std::iterator_traits::value_type; @@ -55,10 +57,18 @@ class SourceGroupInterface : protected SourceInterface { return m_source->getProperty(property_id); } - void setProperty(std::unique_ptr property, const PropertyId& property_id) override { + void setProperty(std::shared_ptr property, const PropertyId& property_id) override { m_source->setProperty(std::move(property), property_id); } + void visitProperties(const PropertyVisitor& visitor) override { + m_source->visitProperties(visitor); + } + + std::unique_ptr clone() const override { + return Euclid::make_unique(m_source->clone()); + } + bool operator<(const SourceWrapper& other) const { return this->m_source < other.m_source; } @@ -108,6 +118,7 @@ class SourceGroupInterface : protected SourceInterface { using SourceInterface::getProperty; using SourceInterface::setProperty; using SourceInterface::setIndexedProperty; + using SourceInterface::clone; }; // end of SourceGroupInterface class diff --git a/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h b/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h index 683a66bda..781a29a65 100644 --- a/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h +++ b/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -71,6 +72,10 @@ class SourceGroupWithOnDemandProperties : public SourceGroupInterface { unsigned int size() const override; + void visitProperties(const PropertyVisitor& visitor) override; + + std::unique_ptr clone() const override; + using SourceInterface::getProperty; using SourceInterface::setProperty; @@ -78,7 +83,7 @@ class SourceGroupWithOnDemandProperties : public SourceGroupInterface { const Property& getProperty(const PropertyId& property_id) const override; - void setProperty(std::unique_ptr property, const PropertyId& property_id) override; + void setProperty(std::shared_ptr property, const PropertyId& property_id) override; private: @@ -97,25 +102,29 @@ class SourceGroupWithOnDemandProperties::EntangledSource : public SourceInterfac public: - EntangledSource(std::shared_ptr source, SourceGroupWithOnDemandProperties& group); + EntangledSource(std::unique_ptr source, SourceGroupWithOnDemandProperties& group); virtual ~EntangledSource() = default; const Property& getProperty(const PropertyId& property_id) const override; - void setProperty(std::unique_ptr property, const PropertyId& property_id) override; + void setProperty(std::shared_ptr property, const PropertyId& property_id) override; + + void visitProperties(const PropertyVisitor& visitor) override; + + std::unique_ptr clone() const override; bool operator<(const EntangledSource& other) const; private: PropertyHolder m_property_holder; - std::shared_ptr m_source; + std::unique_ptr m_source; SourceGroupWithOnDemandProperties& m_group; friend void SourceGroupWithOnDemandProperties::clearGroupProperties(); friend void SourceGroupWithOnDemandProperties::merge(SourceGroupInterface&&); - + friend std::unique_ptr SourceGroupWithOnDemandProperties::clone() const; }; } /* namespace SourceXtractor */ diff --git a/SEFramework/SEFramework/Source/SourceInterface.h b/SEFramework/SEFramework/Source/SourceInterface.h index 3ecb1eeea..9c047a8fc 100644 --- a/SEFramework/SEFramework/Source/SourceInterface.h +++ b/SEFramework/SEFramework/Source/SourceInterface.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -46,6 +47,7 @@ namespace SourceXtractor { class SourceInterface { public: + using PropertyVisitor = std::function&)>; /** * @brief Destructor @@ -73,10 +75,17 @@ class SourceInterface { setIndexedProperty(0, std::forward(args)...); } + /// Call PropertyVisitor once per *set* property. On demand properties are ignored if not yet computed. + /// Concrete implementations must call the visitor from lower to higher preference (i.e. group properties before + /// individual source properties), so in case of conflict the higher priority property can take precedence. + virtual void visitProperties(const PropertyVisitor&) = 0; + /// Returns a reference to the requested property. The property may be computed if needed /// Throws a PropertyNotFoundException if the property cannot be provided. virtual const Property& getProperty(const PropertyId& property_id) const = 0; - virtual void setProperty(std::unique_ptr property, const PropertyId& property_id) = 0; + virtual void setProperty(std::shared_ptr property, const PropertyId& property_id) = 0; + + virtual std::unique_ptr clone() const = 0; }; /* End of SourceInterface class */ diff --git a/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h b/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h index aef4eb8ea..547786080 100644 --- a/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h +++ b/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -50,8 +51,6 @@ class SourceWithOnDemandProperties : public SourceInterface { virtual ~SourceWithOnDemandProperties() = default; // removes copy/move constructors and assignment operators - - SourceWithOnDemandProperties(const SourceWithOnDemandProperties&) = delete; SourceWithOnDemandProperties& operator=(const SourceWithOnDemandProperties&) = delete; SourceWithOnDemandProperties(SourceWithOnDemandProperties&&) = delete; SourceWithOnDemandProperties& operator=(SourceWithOnDemandProperties&&) = delete; @@ -59,20 +58,26 @@ class SourceWithOnDemandProperties : public SourceInterface { /// Constructor explicit SourceWithOnDemandProperties(std::shared_ptr task_provider); + void visitProperties(const PropertyVisitor& visitor) override; + // Note : Because the get/setProperty() methods of the SourceInterface are // templated, the overrides of the non-templated versions will hide them. For // this reason it is necessary to re-introduce the templated methods, which is // done by the using statements below. using SourceInterface::getProperty; using SourceInterface::setProperty; + + std::unique_ptr clone() const override; protected: // Implementation of SourceInterface const Property& getProperty(const PropertyId& property_id) const override; - void setProperty(std::unique_ptr property, const PropertyId& property_id) override; + void setProperty(std::shared_ptr property, const PropertyId& property_id) override; private: + SourceWithOnDemandProperties(const SourceWithOnDemandProperties& other); + std::shared_ptr m_task_provider; PropertyHolder m_property_holder; }; /* End of Source class */ diff --git a/SEFramework/src/lib/Output/OutputRegistry.cpp b/SEFramework/src/lib/Output/OutputRegistry.cpp index 6cef389b1..31132b725 100644 --- a/SEFramework/src/lib/Output/OutputRegistry.cpp +++ b/SEFramework/src/lib/Output/OutputRegistry.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -33,8 +34,9 @@ using namespace Euclid::Table; namespace SourceXtractor { -auto OutputRegistry::getSourceToRowConverter(const std::vector& enabled_properties) -> SourceToRowConverter { - std::vector out_prop_list {}; +auto OutputRegistry::getSourceToRowConverter(const std::vector& enabled_properties) + -> SourceToRowConverter { + std::vector out_prop_list{}; for (auto& prop : enabled_properties) { if (m_output_properties.count(prop) == 0) { throw Elements::Exception() << "Unknown output property " << prop; @@ -47,28 +49,27 @@ auto OutputRegistry::getSourceToRowConverter(const std::vector& ena } } return [this, out_prop_list](const SourceInterface& source) { - std::vector info_list {}; - std::vector cell_values {}; + std::vector info_list{}; + std::vector cell_values{}; for (const auto& property : out_prop_list) { if (m_property_to_names_map.count(property) == 0) { throw Elements::Exception() << "Missing column generator for " << property.name(); } for (const auto& name : m_property_to_names_map.at(property)) { auto& col_info = m_name_to_col_info_map.at(name); - info_list.emplace_back(name, m_name_to_converter_map.at(name).first, - col_info.unit, col_info.description); + info_list.emplace_back(name, m_name_to_converter_map.at(name).first, col_info.unit, col_info.description); cell_values.emplace_back(m_name_to_converter_map.at(name).second(source)); } } if (info_list.empty()) { throw Elements::Exception() << "The given configuration would not generate any output"; } - return Row {std::move(cell_values), std::make_shared(move(info_list))}; + return Row{std::move(cell_values), std::make_shared(move(info_list))}; }; } void OutputRegistry::printPropertyColumnMap(const std::vector& properties) { - std::set properties_set {properties.begin(), properties.end()}; + std::set properties_set{properties.begin(), properties.end()}; for (auto& prop : m_output_properties) { if (!properties.empty() && properties_set.find(prop.first) == properties_set.end()) { continue; @@ -81,12 +82,31 @@ void OutputRegistry::printPropertyColumnMap(const std::vector& prop std::cout << " : " << info.description; } if (info.unit != "") { - std::cout << " " << info.unit << ""; // place here braces "()" around the units, if desired + std::cout << " " << info.unit << ""; // place here braces "()" around the units, if desired } std::cout << '\n'; } } } +std::set OutputRegistry::getColumns(const std::string& property) const { + std::set columns; + auto iter_range = m_output_properties.equal_range(property); + for (auto i = iter_range.first; i != iter_range.second; ++i) { + const auto& attrs = m_property_to_names_map.at(i->second); + std::copy(attrs.begin(), attrs.end(), std::inserter(columns, columns.begin())); + } + + return columns; +} + +std::set OutputRegistry::getColumns(const PropertyId& property) const { + auto iter = m_property_to_names_map.find(property.getTypeId()); + if (iter != m_property_to_names_map.end()) { + return {iter->second.begin(), iter->second.end()}; + } + return {}; } + +} // namespace SourceXtractor diff --git a/SEFramework/src/lib/Pipeline/SourceGrouping.cpp b/SEFramework/src/lib/Pipeline/SourceGrouping.cpp index 8269f262a..09cb50176 100644 --- a/SEFramework/src/lib/Pipeline/SourceGrouping.cpp +++ b/SEFramework/src/lib/Pipeline/SourceGrouping.cpp @@ -104,6 +104,8 @@ void SourceGrouping::receiveProcessSignal(const ProcessSourcesEvent& process_eve sendSource(std::move(*group)); m_source_groups.erase(group); } + + sendProcessSignal(process_event); } std::set SourceGrouping::requiredProperties() const { diff --git a/SEFramework/src/lib/Property/PropertyHolder.cpp b/SEFramework/src/lib/Property/PropertyHolder.cpp index 8436898ca..add189bd8 100644 --- a/SEFramework/src/lib/Property/PropertyHolder.cpp +++ b/SEFramework/src/lib/Property/PropertyHolder.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -37,7 +38,7 @@ const Property& PropertyHolder::getProperty(const PropertyId& property_id) const } } -void PropertyHolder::setProperty(std::unique_ptr property, const PropertyId& property_id) { +void PropertyHolder::setProperty(std::shared_ptr property, const PropertyId& property_id) { m_properties[property_id] = std::move(property); } @@ -49,4 +50,8 @@ void PropertyHolder::clear() { m_properties.clear(); } +void PropertyHolder::update(const SourceXtractor::PropertyHolder& other) { + std::copy(other.m_properties.begin(), other.m_properties.end(), std::inserter(m_properties, m_properties.begin())); +} + } // SEFramework namespace diff --git a/SEFramework/src/lib/Source/EntangledSource.cpp b/SEFramework/src/lib/Source/EntangledSource.cpp index 32db767e0..2b4af1ac1 100644 --- a/SEFramework/src/lib/Source/EntangledSource.cpp +++ b/SEFramework/src/lib/Source/EntangledSource.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -24,16 +25,16 @@ namespace SourceXtractor { -SourceGroupWithOnDemandProperties::EntangledSource::EntangledSource(std::shared_ptr source, SourceGroupWithOnDemandProperties& group) - : m_source(source), m_group(group) { +SourceGroupWithOnDemandProperties::EntangledSource::EntangledSource(std::unique_ptr source, SourceGroupWithOnDemandProperties& group) + : m_source(std::move(source)), m_group(group) { // Normally, it should not be possible that the given source is of type // EntangledSource, because the entangled sources of a group can only be // accessed via the iterator as references. Nevertheless, to be sure that // future changes will not change the behavior, we do a check to the given // source and if it is an EntangledSource we use its encapsulated source instead. - auto entangled_ptr = std::dynamic_pointer_cast(m_source); + auto entangled_ptr = dynamic_cast(m_source.get()); if (entangled_ptr != nullptr) { - m_source = entangled_ptr->m_source; + m_source = std::move(entangled_ptr->m_source); } } @@ -76,10 +77,15 @@ const Property& SourceGroupWithOnDemandProperties::EntangledSource::getProperty( } // end of getProperty() -void SourceGroupWithOnDemandProperties::EntangledSource::setProperty(std::unique_ptr property, const PropertyId& property_id) { +void SourceGroupWithOnDemandProperties::EntangledSource::setProperty(std::shared_ptr property, + const PropertyId& property_id) { m_property_holder.setProperty(std::move(property), property_id); } +std::unique_ptr SourceGroupWithOnDemandProperties::EntangledSource::clone() const { + throw Elements::Exception("Can not clone an entangled source"); +} + bool SourceGroupWithOnDemandProperties::EntangledSource::operator<(const EntangledSource& other) const { return this->m_source < other.m_source; } diff --git a/SEFramework/src/lib/Source/SimpleSourceGroup.cpp b/SEFramework/src/lib/Source/SimpleSourceGroup.cpp index 3884faed7..221d66162 100644 --- a/SEFramework/src/lib/Source/SimpleSourceGroup.cpp +++ b/SEFramework/src/lib/Source/SimpleSourceGroup.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -20,6 +21,10 @@ */ #include "SEFramework/Source/SimpleSourceGroup.h" +#include "AlexandriaKernel/memory_tools.h" +#include + +using Euclid::make_unique; namespace SourceXtractor { @@ -69,7 +74,7 @@ const Property& SimpleSourceGroup::getProperty(const PropertyId& property_id) co return m_property_holder.getProperty(property_id); } -void SimpleSourceGroup::setProperty(std::unique_ptr property, const PropertyId& property_id) { +void SimpleSourceGroup::setProperty(std::shared_ptr property, const PropertyId& property_id) { m_property_holder.setProperty(std::move(property), property_id); } @@ -77,4 +82,19 @@ unsigned int SimpleSourceGroup::size() const { return m_sources.size(); } -} // SourceXtractor namespace +std::unique_ptr SimpleSourceGroup::clone() const { + auto cloned = make_unique(); + for (const auto& src : m_sources) { + cloned->addSource(src.getRef().clone()); + } + cloned->m_property_holder.update(m_property_holder); + return std::unique_ptr(std::move(cloned)); +} + +void SimpleSourceGroup::visitProperties(const PropertyVisitor& visitor) { + std::for_each( + m_property_holder.begin(), m_property_holder.end(), + [visitor](const std::pair>& prop) { visitor(prop.first, prop.second); }); +} + +} // namespace SourceXtractor diff --git a/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp b/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp index c27b71c2d..a8b96a13f 100644 --- a/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp +++ b/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -24,6 +25,9 @@ #include "SEFramework/Source/SourceGroupWithOnDemandProperties.h" #include "SEFramework/Task/GroupTask.h" +#include "AlexandriaKernel/memory_tools.h" + +using Euclid::make_unique; namespace SourceXtractor { @@ -103,7 +107,7 @@ const Property& SourceGroupWithOnDemandProperties::getProperty(const PropertyId& throw PropertyNotFoundException(property_id); } -void SourceGroupWithOnDemandProperties::setProperty(std::unique_ptr property, const PropertyId& property_id) { +void SourceGroupWithOnDemandProperties::setProperty(std::shared_ptr property, const PropertyId& property_id) { m_property_holder.setProperty(std::move(property), property_id); } @@ -118,6 +122,30 @@ unsigned int SourceGroupWithOnDemandProperties::size() const { return m_sources.size(); } +std::unique_ptr SourceGroupWithOnDemandProperties::clone() const { + auto cloned = make_unique(m_task_provider); + for (const auto& source : m_sources) { + auto& entangled_source = dynamic_cast(source.getRef()); + cloned->m_sources.emplace_back(Euclid::make_unique(entangled_source.m_source->clone(), *cloned)); + } + cloned->m_property_holder.update(this->m_property_holder); + return std::unique_ptr(std::move(cloned)); +} + +void SourceGroupWithOnDemandProperties::EntangledSource::visitProperties(const PropertyVisitor& visitor) { + m_group.visitProperties(visitor); + m_source->visitProperties(visitor); + std::for_each( + m_property_holder.begin(), m_property_holder.end(), + [visitor](const std::pair>& prop) { visitor(prop.first, prop.second); }); +} + +void SourceGroupWithOnDemandProperties::visitProperties(const PropertyVisitor& visitor) { + std::for_each( + m_property_holder.begin(), m_property_holder.end(), + [visitor](const std::pair>& prop) { visitor(prop.first, prop.second); }); +} + } // SourceXtractor namespace diff --git a/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp b/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp index 3ca7cc349..9ec9ab77b 100644 --- a/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp +++ b/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -25,7 +26,6 @@ #include "SEFramework/Task/TaskProvider.h" #include "SEFramework/Task/SourceTask.h" #include "SEFramework/Property/PropertyNotFoundException.h" - #include "SEFramework/Source/SourceWithOnDemandProperties.h" namespace SourceXtractor { @@ -36,6 +36,11 @@ SourceWithOnDemandProperties::SourceWithOnDemandProperties(std::shared_ptr property, const PropertyId& property_id) { +void SourceWithOnDemandProperties::setProperty(std::shared_ptr property, const PropertyId& property_id) { // just forward to the ObjectWithProperties implementation m_property_holder.setProperty(std::move(property), property_id); } +std::unique_ptr SourceWithOnDemandProperties::clone() const { + return std::unique_ptr(new SourceWithOnDemandProperties(*this)); +} + +void SourceWithOnDemandProperties::visitProperties(const PropertyVisitor& visitor) { + std::for_each( + m_property_holder.begin(), m_property_holder.end(), + [visitor](const std::pair>& prop) { visitor(prop.first, prop.second); }); +} } // SEFramework namespace diff --git a/SEFramework/tests/src/Source/SourceInterface_test.cpp b/SEFramework/tests/src/Source/SourceInterface_test.cpp index 34700b697..311588741 100644 --- a/SEFramework/tests/src/Source/SourceInterface_test.cpp +++ b/SEFramework/tests/src/Source/SourceInterface_test.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -51,7 +52,13 @@ class MockSourceInterface : public SourceInterface { using SourceInterface::setProperty; MOCK_CONST_METHOD1(getProperty, Property& (const PropertyId& property_id)); - void setProperty(std::unique_ptr, const PropertyId& ) {} + void setProperty(std::shared_ptr, const PropertyId& ) {} + + void visitProperties(const PropertyVisitor&) override {} + + std::unique_ptr clone() const override { + return nullptr; + } }; //----------------------------------------------------------------------------- diff --git a/SEImplementation/CMakeLists.txt b/SEImplementation/CMakeLists.txt index 1b52a93d3..1ef3bc573 100644 --- a/SEImplementation/CMakeLists.txt +++ b/SEImplementation/CMakeLists.txt @@ -96,6 +96,7 @@ elements_add_library(SEImplementation src/lib/Image/*.cpp ${COMMON_SRC} src/lib/Prefetcher/*.cpp + src/lib/Property/*.cpp ${PLUGIN_SRC} ${SE_PYTHON_SRC} LINK_LIBRARIES @@ -110,7 +111,7 @@ elements_add_library(SEImplementation #=============================================================================== # SEImplementation has a part that needs to be reachable from Python. #=============================================================================== -elements_add_python_module(SourceXtractorPy +elements_add_python_module(SEPythonConfig src/lib/PythonConfig/PythonModule.cpp LINK_LIBRARIES SEImplementation INCLUDE_DIRS ${Boost_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS} @@ -222,7 +223,6 @@ elements_add_unit_test(AssocMode_test tests/src/Plugin/AssocMode/AssocMode_test. # elements_install_python_modules() # elements_install_scripts() #=============================================================================== -elements_install_python_modules() #=============================================================================== # Add the elements_install_conf_files macro diff --git a/SEImplementation/SEImplementation/Configuration/OutputConfig.h b/SEImplementation/SEImplementation/Configuration/OutputConfig.h index a6f48427d..95adf6b87 100644 --- a/SEImplementation/SEImplementation/Configuration/OutputConfig.h +++ b/SEImplementation/SEImplementation/Configuration/OutputConfig.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -53,11 +54,11 @@ class OutputConfig : public Euclid::Configuration::Configuration { void initialize(const UserValues& args) override; - std::string getOutputFile(); + std::string getOutputFile() const; - OutputFileFormat getOutputFileFormat(); + OutputFileFormat getOutputFileFormat() const; - const std::vector getOutputProperties(); + const std::vector getOutputProperties() const; size_t getFlushSize() const; diff --git a/SEImplementation/SEImplementation/Configuration/PythonConfig.h b/SEImplementation/SEImplementation/Configuration/PythonConfig.h index e738254c3..ec207ddd4 100644 --- a/SEImplementation/SEImplementation/Configuration/PythonConfig.h +++ b/SEImplementation/SEImplementation/Configuration/PythonConfig.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -14,7 +15,7 @@ * along with this library; if not, write to the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -/* +/* * @file PythonConfig.h * @author Nikolaos Apostolakos */ @@ -28,19 +29,22 @@ namespace SourceXtractor { class PythonConfig : public Euclid::Configuration::Configuration { - + public: - + explicit PythonConfig(long manager_id); - + std::map getProgramOptions() override; void preInitialize(const UserValues& args) override; void initialize(const UserValues& args) override; - + PythonInterpreter& getInterpreter() const; +private: + bool m_capture_output = true; + boost::python::object m_measurement_config; }; } diff --git a/SEImplementation/SEImplementation/Measurement/DummyMeasurement.h b/SEImplementation/SEImplementation/Measurement/DummyMeasurement.h index 84cd40113..f35083d25 100644 --- a/SEImplementation/SEImplementation/Measurement/DummyMeasurement.h +++ b/SEImplementation/SEImplementation/Measurement/DummyMeasurement.h @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -42,6 +43,7 @@ class DummyMeasurement : public Measurement { void startThreads() override {}; void stopThreads() override {}; void synchronizeThreads() override {}; + void cancel() override {}; }; } diff --git a/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h b/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h index 7e46a1390..50f34d860 100644 --- a/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h +++ b/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -24,27 +25,29 @@ #ifndef _SEIMPLEMENTATION_OUTPUT_MULTITHREADEDMEASUREMENT_H_ #define _SEIMPLEMENTATION_OUTPUT_MULTITHREADEDMEASUREMENT_H_ -#include -#include -#include -#include -#include -#include "AlexandriaKernel/ThreadPool.h" #include "AlexandriaKernel/Semaphore.h" +#include "AlexandriaKernel/ThreadPool.h" #include "SEFramework/Pipeline/Measurement.h" +#include +#include +#include +#include +#include namespace SourceXtractor { class MultithreadedMeasurement : public Measurement { public: - using SourceToRowConverter = std::function; MultithreadedMeasurement(SourceToRowConverter source_to_row, const std::shared_ptr& thread_pool, unsigned max_queue_size) - : m_source_to_row(source_to_row), - m_thread_pool(thread_pool), - m_group_counter(0), - m_input_done(false), m_abort_raised(false), m_semaphore(max_queue_size) {} + : m_source_to_row(source_to_row) + , m_thread_pool(thread_pool) + , m_group_counter(0) + , m_input_done(false) + , m_abort_raised(false) + , m_cancel(false) + , m_semaphore(max_queue_size) {} ~MultithreadedMeasurement() override; @@ -55,23 +58,32 @@ class MultithreadedMeasurement : public Measurement { void stopThreads() override; void synchronizeThreads() override; + void cancel() override { + m_cancel = true; + } + private: + using QueuePair = std::pair>; + // We want O(1) for the *lowest* value (received order) + using OutputQueue = std::priority_queue, std::greater>; + static void outputThreadStatic(MultithreadedMeasurement* measurement); - void outputThreadLoop(); + void outputThreadLoop(); - SourceToRowConverter m_source_to_row; + SourceToRowConverter m_source_to_row; std::shared_ptr m_thread_pool; - std::unique_ptr m_output_thread; + std::unique_ptr m_output_thread; - int m_group_counter; - std::atomic_bool m_input_done, m_abort_raised; + int m_group_counter; + std::atomic_bool m_input_done, m_abort_raised, m_cancel; - std::condition_variable m_new_output; - std::list>> m_output_queue; - std::mutex m_output_queue_mutex; - Euclid::Semaphore m_semaphore; + std::condition_variable m_new_output; + OutputQueue m_output_queue; + std::queue> m_event_queue; + std::mutex m_output_queue_mutex; + Euclid::Semaphore m_semaphore; }; -} +} // namespace SourceXtractor #endif /* _SEIMPLEMENTATION_OUTPUT_MULTITHREADEDMEASUREMENT_H_ */ diff --git a/SEImplementation/SEImplementation/Property/PixelCoordinateList.h b/SEImplementation/SEImplementation/Property/PixelCoordinateList.h index 662d787f7..10824d71b 100644 --- a/SEImplementation/SEImplementation/Property/PixelCoordinateList.h +++ b/SEImplementation/SEImplementation/Property/PixelCoordinateList.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -14,7 +15,7 @@ * along with this library; if not, write to the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -/* +/* * @file PixelCoordinateList.h * @author nikoapos */ @@ -31,15 +32,18 @@ namespace SourceXtractor { class PixelCoordinateList : public Property { - + public: - + explicit PixelCoordinateList(std::vector coordinate_list) : m_coordinate_list(std::move(coordinate_list)) { } + PixelCoordinateList(const PixelCoordinateList&) = default; + PixelCoordinateList(PixelCoordinateList&&) = default; + virtual ~PixelCoordinateList() = default; - + const std::vector& getCoordinateList() const { return m_coordinate_list; } @@ -47,11 +51,11 @@ class PixelCoordinateList : public Property { bool contains(const PixelCoordinate& coord) const { return std::find(m_coordinate_list.begin(), m_coordinate_list.end(), coord) != m_coordinate_list.end(); } - + private: std::vector m_coordinate_list; - + }; /* End of PixelCoordinateList class */ } /* namespace SourceXtractor */ diff --git a/SEImplementation/SEImplementation/Property/RegisterConverters.h b/SEImplementation/SEImplementation/Property/RegisterConverters.h new file mode 100644 index 000000000..761650e3f --- /dev/null +++ b/SEImplementation/SEImplementation/Property/RegisterConverters.h @@ -0,0 +1,30 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _SEIMPLEMENTATION_PROPERTY_CONVERTERS_H +#define _SEIMPLEMENTATION_PROPERTY_CONVERTERS_H + +#include "SEFramework/Output/OutputRegistry.h" + +namespace SourceXtractor { + +void registerDefaultConverters(OutputRegistry& registry); + +} + +#endif // _SEIMPLEMENTATION_PROPERTY_CONVERTERS_H diff --git a/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h b/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h index a10e14368..b69a77538 100644 --- a/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h +++ b/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h @@ -1,4 +1,4 @@ -/* +/** * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under @@ -15,7 +15,7 @@ * along with this library; if not, write to the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -/* +/* * @file PythonInterpreter.h * @author Nikolaos Apostolakos */ @@ -34,17 +34,20 @@ namespace SourceXtractor { class PythonInterpreter { - + public: - + static PythonInterpreter& getSingleton(); - + void runFile(const std::string& filename, const std::vector& argv); - void setupContext(); + /// Capture Python's stdout and stderr and pass the output through the logging + void captureOutput(); + + void setupContext(boost::python::object config = {}); virtual ~PythonInterpreter(); - + std::map getMeasurementImages(); std::map getApertures(); @@ -54,23 +57,23 @@ class PythonInterpreter { std::map> getApertureOutputColumns(); std::map getConstantParameters(); - + std::map getFreeParameters(); - + std::map getDependentParameters(); - + std::map getPriors(); - + std::map getConstantModels(); std::map getPointSourceModels(); - + std::map getSersicModels(); - + std::map getExponentialModels(); - + std::map getDeVaucouleursModels(); - + std::map getOnnxModels(); std::map> getFrameModelsMap(); @@ -78,7 +81,7 @@ class PythonInterpreter { std::map getModelFittingParams(); private: - + PythonInterpreter(); std::map getMapFromDict(const char* object_name, diff --git a/SEImplementation/src/lib/Configuration/OutputConfig.cpp b/SEImplementation/src/lib/Configuration/OutputConfig.cpp index 1f9457ade..258f264ad 100644 --- a/SEImplementation/src/lib/Configuration/OutputConfig.cpp +++ b/SEImplementation/src/lib/Configuration/OutputConfig.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -76,13 +77,13 @@ void OutputConfig::preInitialize(const UserValues& args) { void OutputConfig::initialize(const UserValues& args) { m_out_file = args.at(OUTPUT_FILE).as(); - + std::stringstream properties_str {args.at(OUTPUT_PROPERTIES).as()}; std::string name; while (std::getline(properties_str, name, ',')) { m_output_properties.emplace_back(name); } - + auto format_name = boost::to_upper_copy(args.at(OUTPUT_FILE_FORMAT).as()); m_format = format_map.at(format_name); @@ -92,15 +93,15 @@ void OutputConfig::initialize(const UserValues& args) { m_unsorted = !args.at(OUTPUT_SORTED).as(); } -std::string OutputConfig::getOutputFile() { +std::string OutputConfig::getOutputFile() const { return m_out_file; } -OutputConfig::OutputFileFormat OutputConfig::getOutputFileFormat() { +OutputConfig::OutputFileFormat OutputConfig::getOutputFileFormat() const { return m_format; } -const std::vector OutputConfig::getOutputProperties() { +const std::vector OutputConfig::getOutputProperties() const { return m_output_properties; } diff --git a/SEImplementation/src/lib/Configuration/PythonConfig.cpp b/SEImplementation/src/lib/Configuration/PythonConfig.cpp index 347796a85..30d09e184 100644 --- a/SEImplementation/src/lib/Configuration/PythonConfig.cpp +++ b/SEImplementation/src/lib/Configuration/PythonConfig.cpp @@ -1,4 +1,4 @@ -/* +/** * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under @@ -15,13 +15,13 @@ * along with this library; if not, write to the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -/* +/* * @file PythonConfig.cpp * @author Nikolaos Apostolakos */ -#include #include +#include using namespace Euclid::Configuration; namespace po = boost::program_options; @@ -29,10 +29,13 @@ namespace fs = boost::filesystem; namespace { -const std::string PYTHON_CONFIG_FILE { "python-config-file" }; -const std::string PYTHON_ARGV { "python-arg" }; +const std::string PYTHON_CONFIG_FILE{"python-config-file"}; +const std::string PYTHON_ARGV{"python-arg"}; +// These are internal and not exposed to the configuration manager +const std::string PYTHON_CONFIG_OBJ{"python-config-object"}; +const std::string PYTHON_CAPTURE_OUTPUT{"python-capture-output"}; -} +} // namespace namespace SourceXtractor { @@ -41,38 +44,49 @@ PythonConfig::PythonConfig(long manager_id) : Configuration(manager_id) { } std::map PythonConfig::getProgramOptions() { - return {{"Measurement config", { - {PYTHON_CONFIG_FILE.c_str(), po::value()->default_value({}, ""), - "Measurements python configuration file"}, - {PYTHON_ARGV.c_str(), po::value>()->multitoken(), - "Parameters to pass to Python via sys.argv"} - }}}; + return {{"Measurement config", + {{PYTHON_CONFIG_FILE.c_str(), po::value()->default_value({}, ""), + "Measurements python configuration file"}, + {PYTHON_ARGV.c_str(), po::value>()->multitoken(), + "Parameters to pass to Python via sys.argv"}}}}; } - void PythonConfig::preInitialize(const UserValues& args) { - auto filename = args.find(PYTHON_CONFIG_FILE)->second.as(); - if (!filename.empty() && !fs::exists(filename)) { - throw Elements::Exception() << "Python configuration file " << filename - << " does not exist"; + auto filename = args.find(PYTHON_CONFIG_FILE)->second.as(); + auto py_obj_iter = args.find(PYTHON_CONFIG_OBJ); + + if (py_obj_iter != args.end()) { + m_measurement_config = py_obj_iter->second.as(); + } else if (!filename.empty() && !fs::exists(filename)) { + throw Elements::Exception() << "Python configuration file " << filename << " does not exist"; + } + + auto py_capture_output = args.find(PYTHON_CAPTURE_OUTPUT); + if (py_capture_output != args.end()) { + m_capture_output = py_capture_output->second.as(); } } void PythonConfig::initialize(const UserValues& args) { - auto &singleton = PythonInterpreter::getSingleton(); - auto filename = args.find(PYTHON_CONFIG_FILE)->second.as(); - if (!filename.empty()) { - std::vector argv; - if (args.find(PYTHON_ARGV) != args.end()) { - argv = args.find(PYTHON_ARGV)->second.as>(); + auto& singleton = PythonInterpreter::getSingleton(); + if (m_capture_output) { + singleton.captureOutput(); + } + if (!m_measurement_config) { + auto filename = args.find(PYTHON_CONFIG_FILE)->second.as(); + if (!filename.empty()) { + std::vector argv; + if (args.find(PYTHON_ARGV) != args.end()) { + argv = args.find(PYTHON_ARGV)->second.as>(); + } + singleton.runFile(filename, argv); } - singleton.runFile(filename, argv); } - singleton.setupContext(); + singleton.setupContext(m_measurement_config); } PythonInterpreter& PythonConfig::getInterpreter() const { return PythonInterpreter::getSingleton(); } -} // end of namespace SourceXtractor +} // end of namespace SourceXtractor diff --git a/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp b/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp index b29fdecff..12666bca1 100644 --- a/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp +++ b/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -21,18 +22,17 @@ * Author: mschefer */ -#include #include +#include #include -#include "SEImplementation/Plugin/SourceIDs/SourceID.h" #include "SEImplementation/Measurement/MultithreadedMeasurement.h" +#include "SEImplementation/Plugin/SourceIDs/SourceID.h" using namespace SourceXtractor; static Elements::Logging logger = Elements::Logging::getLogger("Multithreading"); - MultithreadedMeasurement::~MultithreadedMeasurement() { if (m_output_thread->joinable()) { m_output_thread->join(); @@ -60,12 +60,10 @@ void MultithreadedMeasurement::synchronizeThreads() { std::unique_lock output_lock(m_output_queue_mutex); if (m_output_queue.empty()) { break; - } - else if (m_thread_pool->checkForException(false)) { + } else if (m_thread_pool->checkForException(false)) { logger.fatal() << "An exception was thrown from a worker thread"; m_thread_pool->checkForException(true); - } - else if (m_thread_pool->activeThreads() == 0) { + } else if (m_thread_pool->activeThreads() == 0) { throw Elements::Exception() << "No active threads and the queue is not empty! Please, report this as a bug"; } } @@ -81,31 +79,32 @@ void MultithreadedMeasurement::receiveSource(std::unique_ptr output_lock(m_output_queue_mutex); - m_output_queue.emplace_back(order_number, std::move(source_group)); + m_output_queue.emplace(order_number, std::move(source_group)); } m_new_output.notify_one(); }; - auto lambda_copyable = [lambda = std::make_shared(std::move(lambda))](){ - (*lambda)(); - }; + auto lambda_copyable = [lambda = std::make_shared(std::move(lambda))]() { (*lambda)(); }; m_thread_pool->submit(lambda_copyable); ++m_group_counter; } -void MultithreadedMeasurement::outputThreadStatic(MultithreadedMeasurement *measurement) { +void MultithreadedMeasurement::outputThreadStatic(MultithreadedMeasurement* measurement) { logger.debug() << "Starting output thread"; try { measurement->outputThreadLoop(); - } - catch (const Elements::Exception& e) { + } catch (const Elements::Exception& e) { logger.fatal() << "Output thread got an exception!"; logger.fatal() << e.what(); if (!measurement->m_abort_raised.exchange(true)) { @@ -117,7 +116,9 @@ void MultithreadedMeasurement::outputThreadStatic(MultithreadedMeasurement *meas } void MultithreadedMeasurement::outputThreadLoop() { - while (m_thread_pool->activeThreads() > 0) { + int next_id = 0; + + while (m_thread_pool->activeThreads() > 0 && !m_cancel) { std::unique_lock output_lock(m_output_queue_mutex); // Wait for something in the output queue @@ -125,19 +126,27 @@ void MultithreadedMeasurement::outputThreadLoop() { m_new_output.wait_for(output_lock, std::chrono::milliseconds(100)); } + // Flush events + while (!m_event_queue.empty() && m_event_queue.front().first <= next_id) { + sendProcessSignal(m_event_queue.front().second); + m_event_queue.pop(); + } + // Process the output queue - while (!m_output_queue.empty()) { - sendSource(std::move(m_output_queue.front().second)); - m_output_queue.pop_front(); + if (!m_output_queue.empty() && m_output_queue.top().first <= next_id) { + auto& next_source = const_cast&>(m_output_queue.top().second); + sendSource(std::move(next_source)); + m_output_queue.pop(); + ++next_id; } - if (m_input_done && m_thread_pool->running() + m_thread_pool->queued() == 0 && - m_output_queue.empty()) { + if (m_input_done && m_thread_pool->running() + m_thread_pool->queued() == 0 && m_output_queue.empty()) { break; } } } void MultithreadedMeasurement::receiveProcessSignal(const ProcessSourcesEvent& event) { - sendProcessSignal(event); + std::unique_lock output_lock(m_output_queue_mutex); + m_event_queue.emplace(m_group_counter, event); } diff --git a/SEImplementation/src/lib/Plugin/DetectionFrameInfo/DetectionFrameInfoPlugin.cpp b/SEImplementation/src/lib/Plugin/DetectionFrameInfo/DetectionFrameInfoPlugin.cpp index d806cc303..4e2b183d2 100644 --- a/SEImplementation/src/lib/Plugin/DetectionFrameInfo/DetectionFrameInfoPlugin.cpp +++ b/SEImplementation/src/lib/Plugin/DetectionFrameInfo/DetectionFrameInfoPlugin.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -17,10 +18,10 @@ #include "SEFramework/Plugin/StaticPlugin.h" +#include "SEImplementation/Image/ImageInterfaceTraits.h" #include "SEImplementation/Plugin/DetectionFrameInfo/DetectionFrameInfo.h" -#include "SEImplementation/Plugin/DetectionFrameInfo/DetectionFrameInfoTaskFactory.h" #include "SEImplementation/Plugin/DetectionFrameInfo/DetectionFrameInfoPlugin.h" -#include "SEImplementation/Image/ImageInterfaceTraits.h" +#include "SEImplementation/Plugin/DetectionFrameInfo/DetectionFrameInfoTaskFactory.h" namespace SourceXtractor { @@ -28,12 +29,13 @@ static StaticPlugin detection_frame_info_plugin; void DetectionFrameInfoPlugin::registerPlugin(PluginAPI& plugin_api) { plugin_api.getTaskFactoryRegistry().registerTaskFactory(); + + plugin_api.getOutputRegistry().registerColumnConverter( + "detection_frame_hdu", [](const DetectionFrameInfo& frame_info) { return frame_info.getHduIndex(); }); } std::string DetectionFrameInfoPlugin::getIdString() const { return "DetectionFrameInfo"; } -} - - +} // namespace SourceXtractor diff --git a/SEImplementation/src/lib/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.cpp b/SEImplementation/src/lib/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.cpp index a9a6614bc..ee37969e8 100644 --- a/SEImplementation/src/lib/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.cpp +++ b/SEImplementation/src/lib/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/* + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -21,24 +22,25 @@ * Author: mschefer */ +#include "SEImplementation/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.h" #include "SEFramework/Plugin/StaticPlugin.h" - #include "SEImplementation/Plugin/DetectionFramePixelValues/DetectionFramePixelValues.h" #include "SEImplementation/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesTaskFactory.h" -#include "SEImplementation/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.h" - namespace SourceXtractor { static StaticPlugin detection_frame_pixel_values_plugin; void DetectionFramePixelValuesPlugin::registerPlugin(PluginAPI& plugin_api) { - plugin_api.getTaskFactoryRegistry().registerTaskFactory(); + auto& output_registry = plugin_api.getOutputRegistry(); + output_registry.registerColumnConverter>( + "detection_pixel_values", [](const DetectionFramePixelValues& pixels) { return pixels.getValues(); }); + plugin_api.getTaskFactoryRegistry() + .registerTaskFactory(); } std::string DetectionFramePixelValuesPlugin::getIdString() const { return ""; } -} - +} // namespace SourceXtractor diff --git a/SEImplementation/src/lib/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.cpp b/SEImplementation/src/lib/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.cpp index 004888d30..44f8720bf 100644 --- a/SEImplementation/src/lib/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.cpp +++ b/SEImplementation/src/lib/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -21,26 +22,51 @@ * Author: mschefer */ +#include "SEImplementation/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.h" +#include "NdArray/Borrowed.h" #include "SEFramework/Plugin/StaticPlugin.h" - #include "SEImplementation/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStamp.h" #include "SEImplementation/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampTaskFactory.h" -#include "SEImplementation/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.h" - namespace SourceXtractor { +using Euclid::NdArray::BorrowedRange; +using Euclid::NdArray::NdArray; + static StaticPlugin detection_frame_source_stamp_plugin; +struct StampConverter { + using DetectionVectorImage = DetectionFrameSourceStamp::DetectionVectorImage; + using PixelType = DetectionVectorImage::PixelType; + + std::function m_getter; + + NdArray operator()(const DetectionFrameSourceStamp& property) const { + auto& stamp = m_getter(property); + auto& data = const_cast&>(stamp.getData()); + std::vector shape{static_cast(stamp.getHeight()), static_cast(stamp.getWidth())}; + return NdArray(shape, BorrowedRange{data.data(), data.size()}); + } +}; + void DetectionFrameSourceStampPlugin::registerPlugin(PluginAPI& plugin_api) { - plugin_api.getTaskFactoryRegistry().registerTaskFactory(); + using PixelType = DetectionImage::PixelType; + + plugin_api.getOutputRegistry().registerColumnConverter>( + "detection_stamp", StampConverter{&DetectionFrameSourceStamp::getStamp}); + plugin_api.getOutputRegistry().registerColumnConverter>( + "detection_filtered_stamp", StampConverter{&DetectionFrameSourceStamp::getFilteredStamp}); + plugin_api.getOutputRegistry().registerColumnConverter>( + "detection_variance_stamp", StampConverter{&DetectionFrameSourceStamp::getVarianceStamp}); + plugin_api.getOutputRegistry().registerColumnConverter>( + "detection_thresholded_stamp", StampConverter{&DetectionFrameSourceStamp::getThresholdedStamp}); + + plugin_api.getTaskFactoryRegistry() + .registerTaskFactory(); } std::string DetectionFrameSourceStampPlugin::getIdString() const { return "DetectionFrameSourceStamp"; } -} - - - +} // namespace SourceXtractor diff --git a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTaskFactory.cpp b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTaskFactory.cpp index f242047e2..91587682d 100644 --- a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTaskFactory.cpp +++ b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTaskFactory.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/* + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free diff --git a/SEImplementation/src/lib/Property/RegisterConverters.cpp b/SEImplementation/src/lib/Property/RegisterConverters.cpp new file mode 100644 index 000000000..131f405c5 --- /dev/null +++ b/SEImplementation/src/lib/Property/RegisterConverters.cpp @@ -0,0 +1,40 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEImplementation/Property/RegisterConverters.h" +#include "SEImplementation/Property/PixelCoordinateList.h" +#include "SEImplementation/Property/SourceId.h" + +using Euclid::NdArray::NdArray; + +namespace SourceXtractor { + +void registerDefaultConverters(OutputRegistry& registry) { + registry.registerColumnConverter>( + "pixel_coordinates", [](const PixelCoordinateList& coordinates) { + const auto& list = coordinates.getCoordinateList(); + NdArray coords({list.size(), 2}); + for (size_t i = 0; i < list.size(); ++i) { + coords.at(i, 0) = list[i].m_x; + coords.at(i, 1) = list[i].m_y; + } + return coords; + }); +} + +} // namespace SourceXtractor diff --git a/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp b/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp index 987a9fa95..c6af93740 100644 --- a/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp +++ b/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp @@ -1,4 +1,4 @@ -/* +/** * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under @@ -50,20 +50,32 @@ PythonInterpreter& PythonInterpreter::getSingleton() { } PythonInterpreter::PythonInterpreter() : m_out_wrapper(stdout_logger), m_err_wrapper(stderr_logger) { - // Python sets its own signal handler for SIGINT (Ctrl+C), so it can throw a KeyboardInterrupt - // Here we are not interested on this behaviour, so we get whatever handler we've got (normally - // the default one) and restore it after initializing the interpreter - struct sigaction sigint_handler; - sigaction(SIGINT, nullptr, &sigint_handler); - - PyImport_AppendInittab("pyston", PYSTON_MODULE_INIT); - Py_Initialize(); + // We may be called *from* Python! + if (!Py_IsInitialized()) { + struct sigaction sigint_handler; + + // Python sets its own signal handler for SIGINT (Ctrl+C), so it can throw a KeyboardInterrupt + // Here we are not interested on this behaviour, so we get whatever handler we've got (normally + // the default one) and restore it after initializing the interpreter + sigaction(SIGINT, nullptr, &sigint_handler); + + // Make pyston importable + PyImport_AppendInittab("pyston", PYSTON_MODULE_INIT); + + // Initialize Python + Py_Initialize(); #if PY_VERSION_HEX < 3090000 - PyEval_InitThreads(); + PyEval_InitThreads(); #endif - PyEval_SaveThread(); + PyEval_SaveThread(); - sigaction(SIGINT, &sigint_handler, nullptr); + // Restore SIGINT handler + sigaction(SIGINT, &sigint_handler, nullptr); + } + + // Import ourselves so the conversions are registered + Pyston::GILLocker locker; + py::import("_SEPythonConfig"); } PythonInterpreter::~PythonInterpreter() { @@ -93,13 +105,6 @@ void PythonInterpreter::runFile(const std::string& filename, const std::vector */ @@ -36,7 +37,7 @@ namespace bp = boost::python; namespace SourceXtractor { -BOOST_PYTHON_MODULE(_SourceXtractorPy) { +BOOST_PYTHON_MODULE(_SEPythonConfig) { bp::class_("OutputWrapper", "A file-like object used to wrap stdout and stderr", bp::no_init) diff --git a/SEMain/src/program/SourceXtractor.cpp b/SEMain/src/program/SourceXtractor.cpp index 49b2d4937..66b20398e 100644 --- a/SEMain/src/program/SourceXtractor.cpp +++ b/SEMain/src/program/SourceXtractor.cpp @@ -73,7 +73,6 @@ #include "SEImplementation/Configuration/WeightImageConfig.h" #include "SEImplementation/Configuration/MemoryConfig.h" #include "SEImplementation/Configuration/OutputConfig.h" -#include "SEImplementation/Configuration/SamplingConfig.h" #include "SEImplementation/CheckImages/CheckImages.h" #include "SEImplementation/Prefetcher/Prefetcher.h" @@ -189,7 +188,6 @@ class SEMain : public Elements::Program { config_manager.registerConfiguration(); config_manager.registerConfiguration(); config_manager.registerConfiguration(); - config_manager.registerConfiguration(); config_manager.registerConfiguration(); CheckImages::getInstance().reportConfigDependencies(config_manager); diff --git a/SEPythonModule/CMakeLists.txt b/SEPythonModule/CMakeLists.txt new file mode 100644 index 000000000..61a225225 --- /dev/null +++ b/SEPythonModule/CMakeLists.txt @@ -0,0 +1,79 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12) + +#=============================================================================== +# Load elements_subdir macro here +# Examples: +# For declaring a project module: +# elements_subdir(ElementsExamples) +#=============================================================================== +elements_subdir(SEPythonModule) + +#=============================================================================== +# Load elements_depends_on_subdirs macro here +# For creating a dependency onto an other accessible module +# elements_depends_on_subdirs(ElementsKernel) +#=============================================================================== +elements_depends_on_subdirs(ElementsKernel) +elements_depends_on_subdirs(SEImplementation) +elements_depends_on_subdirs(SEMain) + +#=============================================================================== +# Add the find_package macro (a pure CMake command) here to locate the +# libraries. +# Examples: +# find_package(CppUnit) +#=============================================================================== +find_package(PythonInterp ${PYTHON_EXPLICIT_VERSION} COMPONENTS Development) +set(PYTHON_SUFFIX "${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}") +find_package(Boost 1.63 COMPONENTS "numpy${PYTHON_SUFFIX}" "python${PYTHON_SUFFIX}") + +if (Boost_FOUND) + #=============================================================================== + # Declare the library dependencies here + # Example: + # elements_add_library(ElementsExamples src/Lib/*.cpp + # INCLUDE_DIRS Boost ElementsKernel + # LINK_LIBRARIES Boost ElementsKernel + # PUBLIC_HEADERS ElementsExamples) + #=============================================================================== + elements_add_python_module(SEPythonModule + src/lib/*.cpp + LINK_LIBRARIES SEImplementation SEMain Boost PythonInterp + ) + + #=============================================================================== + # Declare the executables here + # Example: + # elements_add_executable(ElementsProgramExample src/Program/ProgramExample.cpp + # INCLUDE_DIRS Boost ElementsExamples + # LINK_LIBRARIES Boost ElementsExamples) + #=============================================================================== + + #=============================================================================== + # Declare the Boost tests here + # Example: + # elements_add_unit_test(BoostClassExample tests/src/Boost/ClassExample_test.cpp + # EXECUTABLE BoostClassExample_test + # INCLUDE_DIRS ElementsExamples + # LINK_LIBRARIES ElementsExamples TYPE Boost) + #=============================================================================== + + #=============================================================================== + # Use the following macro for python modules, scripts and aux files: + # elements_install_python_modules() + # elements_install_scripts() + #=============================================================================== + + #=============================================================================== + # Declare the Python programs here + # Examples : + # elements_add_python_program(PythonProgramExample + # ElementsExamples.PythonProgramExample) + #=============================================================================== + + #=============================================================================== + # Add the elements_install_conf_files macro + # Examples: + # elements_install_conf_files() + #=============================================================================== +endif (Boost_FOUND) diff --git a/SEPythonModule/SEPythonModule/ConfigAdapter.h b/SEPythonModule/SEPythonModule/ConfigAdapter.h new file mode 100644 index 000000000..cb789362f --- /dev/null +++ b/SEPythonModule/SEPythonModule/ConfigAdapter.h @@ -0,0 +1,51 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_CONFIGADAPTER_H +#define SOURCEXTRACTORPLUSPLUS_CONFIGADAPTER_H + +#include +#include +#include +#include + +namespace SourceXPy { + +class ConfigAdapter { +public: + using config_map_t = std::map; + + explicit ConfigAdapter(boost::program_options::options_description opt_desc); + + void fromPython(const boost::python::dict& config); + + template + void set(const std::string& key, T&& value) { + m_options[key] = boost::program_options::variable_value(std::forward(value), false); + } + + const config_map_t& getOptions() const; + +private: + boost::program_options::options_description m_option_descriptions; + config_map_t m_options; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_CONFIGADAPTER_H diff --git a/SEPythonModule/SEPythonModule/Context.h b/SEPythonModule/SEPythonModule/Context.h new file mode 100644 index 000000000..856e92678 --- /dev/null +++ b/SEPythonModule/SEPythonModule/Context.h @@ -0,0 +1,109 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_CONTEXT_H +#define SOURCEXTRACTORPLUSPLUS_CONTEXT_H + +#include +#include +#include +#include + +namespace Euclid { +namespace Configuration { +class ConfigManager; +} +} // namespace Euclid + +namespace SourceXtractor { +class TaskFactoryRegistry; +class TaskProvider; +class OutputRegistry; +class PluginManager; +class SegmentationFactory; +class SourceFactory; +class SourceGroupFactory; +class PartitionFactory; +class GroupingFactory; +class DeblendingFactory; +class OutputFactory; +class SourceWithOnDemandPropertiesFactory; +class MeasurementFactory; +class SourceInterface; +} // namespace SourceXtractor + +namespace SourceXPy { + +/** + * Wrap the required context for a sourcextractor++ run + */ +class Context { +public: + Context(const boost::python::dict& global_config, const boost::python::object& measurement_config); + Context(const Context&) = delete; + Context(Context&&) = default; + + boost::python::dict get_properties() const; + + std::shared_ptr m_config_manager; + std::shared_ptr m_task_factory_registry; + std::shared_ptr m_task_provider; + std::shared_ptr m_output_registry; + std::shared_ptr m_plugin_manager; + std::shared_ptr m_source_factory; + std::shared_ptr m_group_factory; + std::shared_ptr m_segmentation_factory; + std::shared_ptr m_partition_factory; + std::shared_ptr m_grouping_factory; + std::shared_ptr m_deblending_factory; + std::shared_ptr m_measurement_factory; + std::shared_ptr m_output_factory; + std::function m_source_to_row; + std::shared_ptr m_thread_pool; + + /// Rethrow an exception if a worker thread failed + void check_exception() const; + + static void enter(const std::shared_ptr&); + static void exit(const std::shared_ptr&, const boost::python::object& exc_type, + const boost::python::object& exc_value, const boost::python::object& traceback); + static const std::shared_ptr& get_global_context(); + +private: + static thread_local std::shared_ptr s_context; +}; + +class ContextPtr { +public: + ContextPtr(std::shared_ptr ptr) : m_ptr(std::move(ptr)) { // NOLINT: Allow implicit conversion + if (!m_ptr) { + m_ptr = Context::get_global_context(); + } + } + + Context* operator->() const { + return m_ptr.get(); + } + +private: + std::shared_ptr m_ptr; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_CONTEXT_H diff --git a/SEPythonModule/SEPythonModule/Deblending.h b/SEPythonModule/SEPythonModule/Deblending.h new file mode 100644 index 000000000..24538f9cc --- /dev/null +++ b/SEPythonModule/SEPythonModule/Deblending.h @@ -0,0 +1,50 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_DEBLENDING_H +#define SOURCEXTRACTORPLUSPLUS_DEBLENDING_H + +#include "SEFramework/Pipeline/Deblending.h" +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEPythonModule/Context.h" + +namespace SourceXPy { + +class Deblending : public SourceXtractor::PipelineReceiver { +public: + explicit Deblending(ContextPtr context); + ~Deblending() override = default; + + std::string repr() const; + + void setNextStage(const boost::python::object& callback); + + void receiveSource(std::unique_ptr source) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj) const; + +private: + ContextPtr m_context; + std::shared_ptr m_deblending; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_DEBLENDING_H diff --git a/SEPythonModule/SEPythonModule/FitsOutput.h b/SEPythonModule/SEPythonModule/FitsOutput.h new file mode 100644 index 000000000..f1f90d4d6 --- /dev/null +++ b/SEPythonModule/SEPythonModule/FitsOutput.h @@ -0,0 +1,56 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_FITSOUTPUT_H +#define SOURCEXTRACTORPLUSPLUS_FITSOUTPUT_H + +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEFramework/Source/SourceGroupInterface.h" +#include "SEPythonModule/Context.h" +#include + +namespace SourceXtractor { +class Output; +} + +namespace SourceXPy { + +class FitsOutput : public SourceXtractor::PipelineReceiver { +public: + explicit FitsOutput(ContextPtr context); + ~FitsOutput() override; + + std::string repr() const; + + void receiveSource(std::unique_ptr group) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj); + + void get(std::chrono::microseconds timeout); + +private: + ContextPtr m_context; + std::shared_ptr m_output; + Euclid::Semaphore m_semaphore{0}; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_FITSOUTPUT_H diff --git a/SEPythonModule/SEPythonModule/Grouping.h b/SEPythonModule/SEPythonModule/Grouping.h new file mode 100644 index 000000000..77c256044 --- /dev/null +++ b/SEPythonModule/SEPythonModule/Grouping.h @@ -0,0 +1,49 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_GROUPING_H +#define SOURCEXTRACTORPLUSPLUS_GROUPING_H + +#include "SEFramework/Pipeline/SourceGrouping.h" +#include "SEPythonModule/Context.h" + +namespace SourceXPy { + +class Grouping : public SourceXtractor::PipelineReceiver { +public: + explicit Grouping(ContextPtr context); + ~Grouping() override = default; + + std::string repr() const; + + void setNextStage(const boost::python::object& callback); + + void receiveSource(std::unique_ptr source) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj) const; + +private: + ContextPtr m_context; + std::shared_ptr m_grouping; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_GROUPING_H diff --git a/SEPythonModule/SEPythonModule/Measurement.h b/SEPythonModule/SEPythonModule/Measurement.h new file mode 100644 index 000000000..55001ca99 --- /dev/null +++ b/SEPythonModule/SEPythonModule/Measurement.h @@ -0,0 +1,51 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_MEASUREMENT_H +#define SOURCEXTRACTORPLUSPLUS_MEASUREMENT_H + +#include "SEFramework/Pipeline/Measurement.h" +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEFramework/Source/SourceGroupInterface.h" +#include "SEPythonModule/Context.h" + +namespace SourceXPy { + +class Measurement : public SourceXtractor::PipelineReceiver { +public: + explicit Measurement(ContextPtr context); + ~Measurement() override; + + std::string repr() const; + + void setNextStage(const boost::python::object& callback); + + void receiveSource(std::unique_ptr group) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj); + +private: + ContextPtr m_context; + std::unique_ptr m_measurement; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_MEASUREMENT_H diff --git a/SEPythonModule/SEPythonModule/NumpyOutput.h b/SEPythonModule/SEPythonModule/NumpyOutput.h new file mode 100644 index 000000000..0c9a0a456 --- /dev/null +++ b/SEPythonModule/SEPythonModule/NumpyOutput.h @@ -0,0 +1,54 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_NUMPYOUTPUT_H +#define SOURCEXTRACTORPLUSPLUS_NUMPYOUTPUT_H + +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEFramework/Source/SourceGroupInterface.h" +#include "SEPythonModule/Context.h" +#include +#include
+ +namespace SourceXPy { + +class NumpyOutput : public SourceXtractor::PipelineReceiver { +public: + explicit NumpyOutput(ContextPtr context); + ~NumpyOutput() override; + + std::string repr() const; + + void receiveSource(std::unique_ptr group) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj); + + boost::python::object getTable(std::chrono::microseconds timeout); + +private: + ContextPtr m_context; + std::function m_source_to_row; + std::vector m_rows; + Euclid::Semaphore m_semaphore{0}; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_NUMPYOUTPUT_H diff --git a/SEPythonModule/SEPythonModule/Partition.h b/SEPythonModule/SEPythonModule/Partition.h new file mode 100644 index 000000000..d48b778f3 --- /dev/null +++ b/SEPythonModule/SEPythonModule/Partition.h @@ -0,0 +1,50 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_PARTITION_H +#define SOURCEXTRACTORPLUSPLUS_PARTITION_H + +#include "SEFramework/Pipeline/Partition.h" +#include "SEPythonModule/Context.h" +#include + +namespace SourceXPy { + +class Partition : public SourceXtractor::PipelineReceiver { +public: + explicit Partition(ContextPtr context); + ~Partition() override = default; + + std::string repr() const; + + void setNextStage(const boost::python::object& callback); + + void receiveSource(std::unique_ptr source) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj) const; + +private: + ContextPtr m_context; + std::shared_ptr m_partition; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_PARTITION_H diff --git a/SEPythonModule/SEPythonModule/PipelineReceiver.h b/SEPythonModule/SEPythonModule/PipelineReceiver.h new file mode 100644 index 000000000..8bc9dd27a --- /dev/null +++ b/SEPythonModule/SEPythonModule/PipelineReceiver.h @@ -0,0 +1,107 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_PIPELINERECEIVER_H +#define SOURCEXTRACTORPLUSPLUS_PIPELINERECEIVER_H + +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEFramework/Pipeline/SourceGrouping.h" +#include "SEPythonModule/SourceInterface.h" +#include +#include +#include + +namespace SourceXPy { + +using SourceReceiverIfce = SourceXtractor::PipelineReceiver; +using GroupReceiverIfce = SourceXtractor::PipelineReceiver; + +std::string ProcessSourcesEventRepr(const SourceXtractor::ProcessSourcesEvent&); + +bool ProcessSourcesEventMustProcess(const SourceXtractor::ProcessSourcesEvent&, const AttachedSource&); + +class AllFramesDone : public SourceXtractor::SelectionCriteria { +public: + bool mustBeProcessed(const SourceXtractor::SourceInterface& source) const override; + + static SourceXtractor::ProcessSourcesEvent create(); +}; + +class PipelineSourceReceiver : public SourceReceiverIfce { +public: + explicit PipelineSourceReceiver(boost::python::object callback, ContextPtr context) + : m_callback(std::move(callback)), m_context(std::move(context)){}; + + ~PipelineSourceReceiver() override = default; + + void receiveSource(std::unique_ptr source) override { + Pyston::GILLocker gil; + try { + m_callback(std::make_shared(m_context, std::move(source))); + } catch (const boost::python::error_already_set&) { + throw Pyston::Exception(); + } + } + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override { + Pyston::GILLocker gil; + try { + m_callback(event); + } catch (const boost::python::error_already_set&) { + throw Pyston::Exception(); + } + } + +private: + boost::python::object m_callback; + ContextPtr m_context; +}; + +class PipelineGroupReceiver : public GroupReceiverIfce { +public: + explicit PipelineGroupReceiver(boost::python::object callback, ContextPtr context) + : m_callback(std::move(callback)), m_context(std::move(context)){}; + + ~PipelineGroupReceiver() override = default; + + void receiveSource(std::unique_ptr source) override { + Pyston::GILLocker gil; + try { + m_callback(std::make_shared(SourceGroup{std::move(source), m_context})); + } catch (const boost::python::error_already_set&) { + throw Pyston::Exception(); + } + } + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override { + Pyston::GILLocker gil; + try { + m_callback(event); + } catch (const boost::python::error_already_set&) { + throw Pyston::Exception(); + } + } + +private: + boost::python::object m_callback; + ContextPtr m_context; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_PIPELINERECEIVER_H diff --git a/SEPythonModule/SEPythonModule/Segmentation.h b/SEPythonModule/SEPythonModule/Segmentation.h new file mode 100644 index 000000000..d86596773 --- /dev/null +++ b/SEPythonModule/SEPythonModule/Segmentation.h @@ -0,0 +1,52 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_SEGMENTATION_H +#define SOURCEXTRACTORPLUSPLUS_SEGMENTATION_H + +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEPythonModule/Context.h" +#include "SEPythonModule/PipelineReceiver.h" +#include + +// Forward declaration +namespace SourceXtractor { +class Segmentation; +} + +namespace SourceXPy { + +class Segmentation { +public: + explicit Segmentation(ContextPtr context); + + std::string repr() const; + + void setNextStage(const boost::python::object& callback); + + void call() const; + +private: + ContextPtr m_context; + std::shared_ptr m_segmentation; + std::shared_ptr m_next_stage; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_SEGMENTATION_H diff --git a/SEPythonModule/SEPythonModule/SourceInterface.h b/SEPythonModule/SEPythonModule/SourceInterface.h new file mode 100644 index 000000000..f4c1a7c6e --- /dev/null +++ b/SEPythonModule/SEPythonModule/SourceInterface.h @@ -0,0 +1,121 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_SOURCEINTERFACE_H +#define SOURCEXTRACTORPLUSPLUS_SOURCEINTERFACE_H + +#include "SEFramework/Source/SourceGroupInterface.h" +#include "SEFramework/Source/SourceInterface.h" +#include "SEImplementation/Property/PixelCoordinateList.h" +#include "SEPythonModule/Context.h" +#include +#include +#include +#include +#include + +namespace SourceXPy { + +/** + * A DetachedSource is "outside" sourcextractor++'s pipeline, so it does not keep any + * reference to internal properties such as the detection frame + */ +struct DetachedSource { + boost::python::dict m_attributes; + + std::string repr() const; + boost::python::list attributes() const; + boost::python::object attribute(const std::string& key) const; +}; + +/** + * An AttachedSource is bound to sourcextractor++'s pipeline. It can not be serialized. + */ +struct AttachedSource { + ContextPtr m_context; + SourceXtractor::SourceInterface* m_source_ptr = nullptr; + + AttachedSource(ContextPtr context, SourceXtractor::SourceInterface* source_ptr) + : m_context(std::move(context)), m_source_ptr(source_ptr) {} + + boost::python::object attribute(const std::string& key) const; + DetachedSource detach() const; +}; + +/** + * An OwnedSource is fully owned by the pipeline stage that receives it. + * It is still attached to the pipeline, but it is safe to keep a reference to it from Python + * @warning This is only true since the pipelines clone the sources that come from Python, which is an inefficiency. + * If acceptable, m_owned_source could be moved away, m_source_ptr set to nullptr, and let any later call + * catch this nullptr situation if the caller kept a reference without explicitly cloning. + */ +struct OwnedSource : public AttachedSource { + OwnedSource(ContextPtr context, std::unique_ptr source) + : AttachedSource(std::move(context), source.get()), m_owned_source(std::move(source)) {} + + std::string repr() const; + + std::unique_ptr m_owned_source; + + static std::shared_ptr create(const std::shared_ptr& context, int detection_frame_idx, + int detection_id, const boost::python::tuple& pixels); +}; + +/** + * An EntangledSource lifetime is bound to its containing SourceGroup. When iterating a SourceGroup, + * only the current EntangledSource is safe to use. i.e. + * + * sources = [] + * for source in group: + * print(source) # Safe + * sources.append(source) # Unsafe + * print(sources) # Unsafe + */ +struct EntangledSource : public AttachedSource { + explicit EntangledSource(ContextPtr context) : AttachedSource(std::move(context), nullptr) {} + + std::string repr() const; +}; + +/** + * A SourceGroup is always owned by the called pipeline stage + */ +struct SourceGroup { + struct Iterator { + SourceXtractor::SourceGroupInterface::const_iterator m_i; + SourceXtractor::SourceGroupInterface::const_iterator m_end; + std::shared_ptr m_holder; + + // It always returns the same! + std::shared_ptr next(); + }; + + std::unique_ptr m_group; + ContextPtr m_context; + + std::string repr() const; + size_t size() const; + boost::python::object attribute(const std::string& key) const; + Iterator iter() const; + + static std::shared_ptr create(const std::shared_ptr& context, boost::python::list& sources); +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_SOURCEINTERFACE_H diff --git a/SEPythonModule/src/lib/ConfigAdapter.cpp b/SEPythonModule/src/lib/ConfigAdapter.cpp new file mode 100644 index 000000000..db8ad4ab8 --- /dev/null +++ b/SEPythonModule/src/lib/ConfigAdapter.cpp @@ -0,0 +1,62 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/ConfigAdapter.h" +#include +#include + +namespace SourceXPy { + +namespace py = boost::python; +namespace po = boost::program_options; + +ConfigAdapter::ConfigAdapter(po::options_description opt_desc) : m_option_descriptions(std::move(opt_desc)) { + // Populate defaults + for (const auto& p : m_option_descriptions.options()) { + boost::any default_value; + if (p->semantic()->apply_default(default_value)) { + m_options[p->long_name()] = boost::program_options::variable_value(default_value, true); + } + } +} + +void ConfigAdapter::fromPython(const py::dict& config) { + auto pairs = config.items(); + for (ssize_t i = 0; i < len(pairs); ++i) { + py::tuple pair(pairs[i]); + std::string key = py::extract(pair[0]); + // Dirty trick, serialize into string and let boost options parse back to whatever is expected + // This is necessary because we can not easily recover the actual expected type unless there is + // a default, and we want to work also in corner cases as when an integer-like or double is used + // to setup a float + std::string value = py::extract(pair[1].attr("__str__")()); + + const auto opt = m_option_descriptions.find_nothrow(key, false); + if (opt) { + boost::any opt_value; + opt->semantic()->parse(opt_value, {value}, true); + m_options[opt->long_name()] = boost::program_options::variable_value(opt_value, false); + } + } +} + +auto ConfigAdapter::getOptions() const -> const config_map_t& { + return m_options; +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/Context.cpp b/SEPythonModule/src/lib/Context.cpp new file mode 100644 index 000000000..315e3a7d1 --- /dev/null +++ b/SEPythonModule/src/lib/Context.cpp @@ -0,0 +1,151 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Context.h" +#include "SEFramework/Output/OutputRegistry.h" +#include "SEFramework/Plugin/PluginManager.h" +#include "SEFramework/Source/SourceGroupWithOnDemandPropertiesFactory.h" +#include "SEFramework/Source/SourceWithOnDemandPropertiesFactory.h" +#include "SEFramework/Task/TaskFactoryRegistry.h" +#include "SEFramework/Task/TaskProvider.h" +#include "SEImplementation/Configuration/DetectionFrameConfig.h" +#include "SEImplementation/Configuration/MultiThreadingConfig.h" +#include "SEImplementation/Configuration/OutputConfig.h" +#include "SEImplementation/Configuration/PythonConfig.h" +#include "SEImplementation/Deblending/DeblendingFactory.h" +#include "SEImplementation/Grouping/GroupingFactory.h" +#include "SEImplementation/Measurement/MeasurementFactory.h" +#include "SEImplementation/Output/OutputFactory.h" +#include "SEImplementation/Partition/PartitionFactory.h" +#include "SEImplementation/Property/RegisterConverters.h" +#include "SEImplementation/Segmentation/SegmentationFactory.h" +#include "SEMain/SourceXtractorConfig.h" +#include "SEPythonModule/ConfigAdapter.h" +#include +#include +#include + +namespace SourceXPy { + +namespace Configuration = Euclid::Configuration; +namespace py = boost::python; +namespace se = SourceXtractor; +using Euclid::Configuration::ConfigManager; + +thread_local std::shared_ptr Context::s_context = nullptr; + +Context::Context(const py::dict& global_config, const py::object& measurement_config) + : m_task_factory_registry(std::make_shared()) + , m_task_provider(std::make_shared(m_task_factory_registry)) + , m_output_registry(std::make_shared()) + , m_plugin_manager(std::make_shared(m_task_factory_registry, m_output_registry, "", + std::vector{})) + , m_source_factory(std::make_shared(m_task_provider)) + , m_group_factory(std::make_shared(m_task_provider)) + , m_segmentation_factory(std::make_shared(m_task_provider)) + , m_partition_factory(std::make_shared(m_source_factory)) + , m_grouping_factory(std::make_shared(m_group_factory)) + , m_deblending_factory(std::make_shared(m_source_factory)) + , m_measurement_factory(std::make_shared(m_output_registry)) + , m_output_factory(std::make_shared(m_output_registry)) { + + // This is *very* risky, but the only alternative to a big refactoring + m_config_manager.reset( + &ConfigManager::getInstance(Configuration::getUniqueManagerId()), + [](const ConfigManager* manager) { Euclid::Configuration::ConfigManager::deregisterInstance(manager->getId()); }); + + m_plugin_manager->loadPlugins(); + + // Register configuration + m_segmentation_factory->reportConfigDependencies(*m_config_manager); + m_partition_factory->reportConfigDependencies(*m_config_manager); + m_grouping_factory->reportConfigDependencies(*m_config_manager); + m_deblending_factory->reportConfigDependencies(*m_config_manager); + m_plugin_manager->getTaskFactoryRegistry().reportConfigDependencies(*m_config_manager); + m_measurement_factory->reportConfigDependencies(*m_config_manager); + m_output_factory->reportConfigDependencies(*m_config_manager); + + m_config_manager->registerConfiguration(); + m_config_manager->registerConfiguration(); + m_config_manager->registerConfiguration(); + + // Configure + ConfigAdapter config_wrapper(m_config_manager->closeRegistration()); + config_wrapper.fromPython(global_config); + auto options = config_wrapper.getOptions(); + + // Override the python object to use for the measurement configuration and disable output capture + options["python-config-object"].value() = boost::any(measurement_config); + options["python-capture-output"].value() = boost::any(false); + + m_config_manager->initialize(options); + + m_segmentation_factory->configure(*m_config_manager); + m_partition_factory->configure(*m_config_manager); + m_grouping_factory->configure(*m_config_manager); + m_deblending_factory->configure(*m_config_manager); + m_task_factory_registry->configure(*m_config_manager); + m_measurement_factory->configure(*m_config_manager); + m_output_factory->configure(*m_config_manager); + + // Get the thread pool + m_thread_pool = m_config_manager->getConfiguration().getThreadPool(); + + // Register the output properties + m_task_factory_registry->registerPropertyInstances(*m_output_registry); + + // Get hold of the sources-to-row converter + auto enabled_properties = m_config_manager->getConfiguration().getOutputProperties(); + m_source_to_row = m_output_registry->getSourceToRowConverter(enabled_properties); + se::registerDefaultConverters(*m_output_registry); +} + +py::dict Context::get_properties() const { + py::dict properties; + const auto property_names = m_output_registry->getOutputPropertyNames(); + for (auto& prop_name : property_names) { + const auto column_names = m_output_registry->getColumns(prop_name); + py::list attr_names; + boost::for_each(column_names, [&attr_names](const std::string& c) { attr_names.append(c); }); + properties[prop_name] = attr_names; + } + return properties; +} + +void Context::check_exception() const { + m_thread_pool->checkForException(true); +} + +void Context::enter(const std::shared_ptr& context) { + s_context = context; +} + +void Context::exit(const std::shared_ptr&, [[maybe_unused]] const py::object& exc_type, + [[maybe_unused]] const py::object& exc_value, [[maybe_unused]] const py::object& traceback) { + s_context.reset(); +} + +const std::shared_ptr& Context::get_global_context() { + if (!s_context) { + PyErr_SetString(PyExc_RuntimeError, "Need an explicit, or an active Context instance"); + py::throw_error_already_set(); + } + return s_context; +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/Deblending.cpp b/SEPythonModule/src/lib/Deblending.cpp new file mode 100644 index 000000000..43d93d70b --- /dev/null +++ b/SEPythonModule/src/lib/Deblending.cpp @@ -0,0 +1,71 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Deblending.h" +#include "Pyston/GIL.h" +#include "SEImplementation/Deblending/DeblendingFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include + +namespace py = boost::python; +namespace se = SourceXtractor; + +namespace SourceXPy { + +Deblending::Deblending(ContextPtr context) : m_context(std::move(context)) { + m_deblending = m_context->m_deblending_factory->createDeblending(); +} + +std::string Deblending::repr() const { + return "Deblending"; +} + +void Deblending::setNextStage(const py::object& callback) { + m_deblending->setNextStage(std::make_shared(callback, m_context)); +} + +void Deblending::receiveSource(std::unique_ptr source) { + m_deblending->receiveSource(std::move(source)); +} + +void Deblending::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + m_deblending->receiveProcessSignal(event); +} + +void Deblending::call(const py::object& obj) const { + py::extract group_wrapper(obj); + if (group_wrapper.check()) { + const auto& group_ptr = group_wrapper().m_group; + Pyston::SaveThread save_thread; + std::unique_ptr cloned_group_ptr( + dynamic_cast(group_ptr->clone().release())); + m_deblending->receiveSource(std::move(cloned_group_ptr)); + return; + } + py::extract event_wrapper(obj); + if (event_wrapper.check()) { + const auto& event = event_wrapper(); + Pyston::SaveThread save_thread; + m_deblending->receiveProcessSignal(event); + return; + } + PyErr_SetString(PyExc_TypeError, "Deblending: Unexpected python object received"); + py::throw_error_already_set(); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/FitsOutput.cpp b/SEPythonModule/src/lib/FitsOutput.cpp new file mode 100644 index 000000000..6640d28c9 --- /dev/null +++ b/SEPythonModule/src/lib/FitsOutput.cpp @@ -0,0 +1,92 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/FitsOutput.h" +#include "SEImplementation/Output/OutputFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include "SEPythonModule/SourceInterface.h" +#include + +namespace py = boost::python; +namespace se = SourceXtractor; + +namespace SourceXPy { + +FitsOutput::FitsOutput(SourceXPy::ContextPtr context) : m_context(std::move(context)) { + m_output = m_context->m_output_factory->createOutput(); +} + +FitsOutput::~FitsOutput() = default; + +std::string FitsOutput::repr() const { + return "FitsOutput"; +} + +void FitsOutput::receiveSource(std::unique_ptr group) { + m_output->receiveSource(std::move(group)); +} + +void FitsOutput::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + m_output->receiveProcessSignal(event); + auto all_done = std::dynamic_pointer_cast(event.m_selection_criteria); + if (all_done) { + m_semaphore.release(); + } +} + +void FitsOutput::call(const boost::python::object& obj) { + py::extract group(obj); + if (group.check()) { + const auto& group_ptr = group().m_group; + Pyston::SaveThread save_thread; + std::unique_ptr cloned_group_ptr( + dynamic_cast(group_ptr->clone().release())); + m_output->receiveSource(std::move(cloned_group_ptr)); + return; + } + py::extract event_wrapper(obj); + if (event_wrapper.check()) { + const auto& event = event_wrapper(); + Pyston::SaveThread save_thread; + m_output->receiveProcessSignal(event); + return; + } + PyErr_SetString(PyExc_TypeError, "FitsOutput: Unexpected python object received"); + py::throw_error_already_set(); +} + +void FitsOutput::get(std::chrono::microseconds timeout) { + static constexpr std::chrono::seconds try_wait(1); + Pyston::SaveThread save_thread; + while (!m_semaphore.try_acquire_for(try_wait)) { + m_context->m_thread_pool->checkForException(true); + timeout -= try_wait; + if (timeout <= std::chrono::microseconds::zero()) { + Pyston::GILLocker gil; +#if PY_VERSION_HEX >= 0x03030000 + PyErr_SetString(PyExc_TimeoutError, "sourcextractor timed-out"); +#else + PyErr_SetString(PyExc_RuntimeError, "sourcextractor timed-out"); +#endif + py::throw_error_already_set(); + } + } + m_context->m_thread_pool->checkForException(true); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/Grouping.cpp b/SEPythonModule/src/lib/Grouping.cpp new file mode 100644 index 000000000..2675c25e1 --- /dev/null +++ b/SEPythonModule/src/lib/Grouping.cpp @@ -0,0 +1,73 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Grouping.h" +#include "SEImplementation/Grouping/GroupingFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include + +namespace py = boost::python; +namespace se = SourceXtractor; + +namespace SourceXPy { + +Grouping::Grouping(ContextPtr context) : m_context(std::move(context)) { + m_grouping = std::dynamic_pointer_cast(m_context->m_grouping_factory->createGrouping()); +} + +std::string Grouping::repr() const { + return "Grouping"; +} + +void Grouping::setNextStage(const py::object& callback) { + py::extract> cpp_ifce(callback); + if (cpp_ifce.check()) { + m_grouping->setNextStage(cpp_ifce()); + } else { + m_grouping->setNextStage(std::make_shared(callback, m_context)); + } +} + +void Grouping::receiveSource(std::unique_ptr source) { + m_grouping->receiveSource(std::move(source)); +} + +void Grouping::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + m_grouping->receiveProcessSignal(event); +} + +void Grouping::call(const py::object& obj) const { + py::extract source_wrapper(obj); + if (source_wrapper.check()) { + const auto& source_ptr = source_wrapper().m_source_ptr; + Pyston::SaveThread save_thread; + m_grouping->receiveSource(source_ptr->clone()); + return; + } + py::extract event_wrapper(obj); + if (event_wrapper.check()) { + const auto& event = event_wrapper(); + Pyston::SaveThread save_thread; + m_grouping->receiveProcessSignal(event); + return; + } + PyErr_SetString(PyExc_TypeError, "Grouping: Unexpected python object received"); + py::throw_error_already_set(); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/Measurement.cpp b/SEPythonModule/src/lib/Measurement.cpp new file mode 100644 index 000000000..ffb95999c --- /dev/null +++ b/SEPythonModule/src/lib/Measurement.cpp @@ -0,0 +1,85 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Measurement.h" +#include "SEImplementation/Measurement/MeasurementFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include + +namespace py = boost::python; +namespace se = SourceXtractor; + +namespace SourceXPy { + +Measurement::Measurement(ContextPtr context) : m_context(std::move(context)) { + m_measurement = m_context->m_measurement_factory->getMeasurement(); +} + +Measurement::~Measurement() { + try { + m_measurement->cancel(); + m_measurement->stopThreads(); + } catch (...) { + // Must not throw from a destructor + } +} + +std::string Measurement::repr() const { + return "Measurement"; +} + +void Measurement::setNextStage(const boost::python::object& callback) { + py::extract> cpp_ifce(callback); + if (cpp_ifce.check()) { + m_measurement->setNextStage(cpp_ifce()); + } else { + m_measurement->setNextStage(std::make_shared(callback, m_context)); + } + m_measurement->startThreads(); +} + +void Measurement::receiveSource(std::unique_ptr group) { + m_measurement->receiveSource(std::move(group)); +} + +void Measurement::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + m_measurement->receiveProcessSignal(event); +} + +void Measurement::call(const py::object& obj) { + py::extract group(obj); + if (group.check()) { + const auto& group_ptr = group().m_group; + Pyston::SaveThread save_thread; + std::unique_ptr cloned_group_ptr( + dynamic_cast(group_ptr->clone().release())); + m_measurement->receiveSource(std::move(cloned_group_ptr)); + return; + } + py::extract event_wrapper(obj); + if (event_wrapper.check()) { + const auto& event = event_wrapper(); + Pyston::SaveThread save_thread; + m_measurement->receiveProcessSignal(event); + return; + } + PyErr_SetString(PyExc_TypeError, "Measurement: Unexpected python object received"); + py::throw_error_already_set(); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/NumpyOutput.cpp b/SEPythonModule/src/lib/NumpyOutput.cpp new file mode 100644 index 000000000..e56370e28 --- /dev/null +++ b/SEPythonModule/src/lib/NumpyOutput.cpp @@ -0,0 +1,93 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/NumpyOutput.h" +#include "Configuration/ConfigManager.h" +#include "Pyston/Table2Numpy.h" +#include "SEImplementation/Measurement/MeasurementFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include "SEPythonModule/SourceInterface.h" +#include +#include + +using Euclid::Table::Row; +using Euclid::Table::Table; +namespace py = boost::python; +namespace se = SourceXtractor; + +namespace SourceXPy { + +NumpyOutput::NumpyOutput(ContextPtr context) : m_context(std::move(context)) { + m_source_to_row = m_context->m_source_to_row; +} + +NumpyOutput::~NumpyOutput() {} + +std::string NumpyOutput::repr() const { + return "Output"; +} + +void NumpyOutput::receiveSource(std::unique_ptr group) { + boost::transform(*group, std::back_inserter(m_rows), m_source_to_row); +} + +void NumpyOutput::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + auto all_done = std::dynamic_pointer_cast(event.m_selection_criteria); + if (all_done) { + m_semaphore.release(); + } +} + +void NumpyOutput::call(const py::object& obj) { + py::extract group(obj); + if (group.check()) { + const auto& group_ptr = group().m_group; + Pyston::SaveThread save_thread; + boost::transform(*group_ptr, std::back_inserter(m_rows), m_source_to_row); + return; + } + py::extract event_wrapper(obj); + if (!event_wrapper.check()) { + PyErr_SetString(PyExc_TypeError, "NumpyOutput: Unexpected python object received"); + py::throw_error_already_set(); + } +} + +py::object NumpyOutput::getTable(std::chrono::microseconds timeout) { + static constexpr std::chrono::seconds try_wait(1); + { + Pyston::SaveThread save_thread; + while (!m_semaphore.try_acquire_for(try_wait)) { + m_context->m_thread_pool->checkForException(true); + timeout -= try_wait; + if (timeout <= std::chrono::microseconds::zero()) { + Pyston::GILLocker gil; +#if PY_VERSION_HEX >= 0x03030000 + PyErr_SetString(PyExc_TimeoutError, "sourcextractor timed-out"); +#else + PyErr_SetString(PyExc_RuntimeError, "sourcextractor timed-out"); +#endif + py::throw_error_already_set(); + } + } + } + m_context->m_thread_pool->checkForException(true); + return Pyston::table2numpy(Table(m_rows)); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/Partition.cpp b/SEPythonModule/src/lib/Partition.cpp new file mode 100644 index 000000000..a19859e7b --- /dev/null +++ b/SEPythonModule/src/lib/Partition.cpp @@ -0,0 +1,73 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Partition.h" +#include "SEImplementation/Partition/PartitionFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include + +namespace py = boost::python; +namespace se = SourceXtractor; + +namespace SourceXPy { + +Partition::Partition(ContextPtr context) : m_context(std::move(context)) { + m_partition = m_context->m_partition_factory->getPartition(); +} + +std::string Partition::repr() const { + return "Partition"; +} + +void Partition::setNextStage(const py::object& callback) { + py::extract> cpp_ifce(callback); + if (cpp_ifce.check()) { + m_partition->setNextStage(cpp_ifce()); + } else { + m_partition->setNextStage(std::make_shared(callback, m_context)); + } +} + +void Partition::receiveSource(std::unique_ptr source) { + m_partition->receiveSource(std::move(source)); +} + +void Partition::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + m_partition->receiveProcessSignal(event); +} + +void Partition::call(const py::object& obj) const { + py::extract source_wrapper(obj); + if (source_wrapper.check()) { + const auto& source_ptr = source_wrapper().m_source_ptr; + Pyston::SaveThread save_thread; + m_partition->receiveSource(source_ptr->clone()); + return; + } + py::extract event_wrapper(obj); + if (event_wrapper.check()) { + const auto& event = event_wrapper(); + Pyston::SaveThread save_thread; + m_partition->receiveProcessSignal(event); + return; + } + PyErr_SetString(PyExc_TypeError, "Partition: Unexpected python object received"); + py::throw_error_already_set(); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/PipelineReceiver.cpp b/SEPythonModule/src/lib/PipelineReceiver.cpp new file mode 100644 index 000000000..27c2e55e5 --- /dev/null +++ b/SEPythonModule/src/lib/PipelineReceiver.cpp @@ -0,0 +1,47 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/PipelineReceiver.h" + +namespace se = SourceXtractor; + +namespace SourceXPy { + +std::string ProcessSourcesEventRepr(const se::ProcessSourcesEvent& event) { + auto& ptr = event.m_selection_criteria; + if (std::dynamic_pointer_cast(ptr)) { + return "SelectAllCriteria"; + } else if (std::dynamic_pointer_cast(ptr)) { + return "AllFramesDone"; + } + return "LineSelectionCriteria"; +} + +bool ProcessSourcesEventMustProcess(const SourceXtractor::ProcessSourcesEvent& event, const AttachedSource& source) { + return event.m_selection_criteria->mustBeProcessed(*source.m_source_ptr); +} + +bool AllFramesDone::mustBeProcessed(const se::SourceInterface&) const { + return true; +} + +se::ProcessSourcesEvent AllFramesDone::create() { + return se::ProcessSourcesEvent{std::make_shared()}; +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/SePyMain.cpp b/SEPythonModule/src/lib/SePyMain.cpp new file mode 100644 index 000000000..fb0d2f805 --- /dev/null +++ b/SEPythonModule/src/lib/SePyMain.cpp @@ -0,0 +1,226 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Context.h" +#include "SEPythonModule/Deblending.h" +#include "SEPythonModule/FitsOutput.h" +#include "SEPythonModule/Grouping.h" +#include "SEPythonModule/Measurement.h" +#include "SEPythonModule/NumpyOutput.h" +#include "SEPythonModule/Partition.h" +#include "SEPythonModule/PipelineReceiver.h" +#include "SEPythonModule/Segmentation.h" +#include "SEPythonModule/SourceInterface.h" +#include +#include +#include +#include +#include +#include +#include + +using namespace SourceXPy; +namespace py = boost::python; +namespace np = boost::python::numpy; +namespace se = SourceXtractor; + +namespace { + +/** + * Pickle detached sources + */ +struct PickleDetachedSource : public py::pickle_suite { + static py::tuple getinitargs(const DetachedSource&) { + return py::make_tuple(); + } + + static py::tuple getstate(const DetachedSource& source) { + return py::make_tuple(source.m_attributes); + } + + static void setstate(DetachedSource& source, py::tuple state) { + source.m_attributes = py::extract(state[0]); + } +}; + +/** + * Convert std::chrono::duration to datetime.timedelta + */ +struct timedelta_from_std_duration { + timedelta_from_std_duration() { + py::to_python_converter(); + } + + static PyObject* convert(std::chrono::microseconds us) { + auto seconds = std::chrono::duration_cast(us); + us -= seconds; + auto hours = std::chrono::duration_cast(seconds); + seconds -= hours; + return PyDelta_FromDSU(hours.count() / 24, seconds.count(), us.count()); + } +}; + +/** + * Convert datetime.timedelta to std::chrono::duration + */ +struct std_duration_from_timedelta { + std_duration_from_timedelta() { + py::converter::registry::push_back(&convertible, &construct, py::type_id()); + } + + static void* convertible(PyObject* obj_ptr) { + if (PyDelta_Check(obj_ptr)) { + return obj_ptr; + } + return nullptr; + } + + static void construct(PyObject* obj_ptr, py::converter::rvalue_from_python_stage1_data* data) { + auto timedelta = reinterpret_cast(obj_ptr); + auto hours = std::chrono::hours(timedelta->days * 24l); + auto seconds = std::chrono::seconds(timedelta->seconds); + auto microseconds = std::chrono::microseconds(timedelta->microseconds); + auto duration = hours + seconds + microseconds; + + auto storage = + (reinterpret_cast*>(data))->storage.bytes; + new (storage) std::chrono::microseconds(duration); + data->convertible = storage; + } +}; + +void translate_pyston_exc(const Pyston::Exception& exc) { + exc.restore(); +} + +} // namespace + +BOOST_PYTHON_MODULE(_SEPythonModule) { + np::initialize(); + py::object None; + + PyDateTime_IMPORT; + std_duration_from_timedelta(); + timedelta_from_std_duration(); + + py::class_( + "Context", py::init((py::arg("config"), py::arg("measurement_config") = py::object()))) + .def("get_properties", &Context::get_properties) + .def("check_exception", &Context::check_exception) + .def("__enter__", &Context::enter) + .def("__exit__", &Context::exit); + py::register_ptr_to_python>(); + + py::class_("DetachedSource") + .def("__repr__", &DetachedSource::repr) + .def("__getattr__", &DetachedSource::attribute) + .def("__dir__", &DetachedSource::attributes) + .def_pickle(PickleDetachedSource()); + + py::class_("Source", py::no_init) + .def("__getattr__", &AttachedSource::attribute) + .def("detach", &AttachedSource::detach) + .def("create", &OwnedSource::create, py::args("context", "detection_frame", "detection_id", "pixel_coordinates")) + .staticmethod("create"); + py::register_ptr_to_python>(); + + py::class_, boost::noncopyable>("OwnedSource", py::no_init) + .def("__repr__", &OwnedSource::repr); + py::register_ptr_to_python>(); + + py::class_, boost::noncopyable>("EntangledSource", py::no_init) + .def("__repr__", &EntangledSource::repr); + + py::register_ptr_to_python>(); + + py::class_("GroupIterator", py::no_init).def("__next__", &SourceGroup::Iterator::next); + + py::class_("Group", py::no_init) + .def("__repr__", &SourceGroup::repr) + .def("__getattr__", &SourceGroup::attribute) + .def("__len__", &SourceGroup::size) + .def("__iter__", &SourceGroup::iter) + .def("create", &SourceGroup::create) + .staticmethod("create"); + py::register_ptr_to_python>(); + + py::class_("ProcessSourcesEvent", py::no_init) + .def("__repr__", &ProcessSourcesEventRepr) + .def("must_process", &ProcessSourcesEventMustProcess); + + py::class_("SourceReceiver", py::no_init); + py::class_("GroupReceiverIfce", py::no_init); + + py::class_("Segmentation", py::init>((py::arg("context") = None))) + .def("__repr__", &Segmentation::repr) + .def("set_next_stage", &Segmentation::setNextStage) + .def("__call__", &Segmentation::call); + + py::class_>("Partition", + py::init>((py::arg("context") = None))) + .def("__repr__", &Partition::repr) + .def("set_next_stage", &Partition::setNextStage) + .def("__call__", &Partition::call); + + py::class_>("Grouping", + py::init>((py::arg("context") = None))) + .def("__repr__", &Grouping::repr) + .def("set_next_stage", &Grouping::setNextStage) + .def("__call__", &Grouping::call); + + py::class_>("Deblending", + py::init>((py::arg("context") = None))) + .def("__repr__", &Deblending::repr) + .def("set_next_stage", &Deblending::setNextStage) + .def("__call__", &Deblending::call); + + py::class_, boost::noncopyable>( + "Measurement", py::init>((py::arg("context") = None))) + .def("__repr__", &Measurement::repr) + .def("set_next_stage", &Measurement::setNextStage) + .def("__call__", &Measurement::call); + + py::class_, boost::noncopyable>( + "NumpyOutput", py::init>((py::arg("Context") = None))) + .def("__repr__", &NumpyOutput::repr) + .def("__call__", &NumpyOutput::call) + .def("get", &NumpyOutput::getTable, (py::arg("timeout") = std::chrono::microseconds::max())); + + py::class_, boost::noncopyable>( + "FitsOutput", py::init>((py::arg("Context") = None))) + .def("__repr__", &FitsOutput::repr) + .def("__call__", &FitsOutput::call) + .def("get", &FitsOutput::get, (py::arg("timeout") = std::chrono::microseconds::max())); + + // For custom segmentation + py::def("AllFramesDone", &AllFramesDone::create); + + // Pyston::Exception wrap Python errors, so unwrap them at the Python boundary + py::register_exception_translator(&translate_pyston_exc); + + // Import pyston into the interpreter so it is importable without tweaking PYTHONPATH +#if PY_MAJOR_VERSION >= 3 + PyObject* pyston = PyInit_pyston(); +#else + initpyston(); + PyObject* pyston = PyImport_ImportModule("_SEPythonConfig"); +#endif + PyObject* modules = PyImport_GetModuleDict(); + PyDict_SetItemString(modules, "pyston", pyston); + Py_DECREF(pyston); +} diff --git a/SEPythonModule/src/lib/Segmentation.cpp b/SEPythonModule/src/lib/Segmentation.cpp new file mode 100644 index 000000000..58ec57802 --- /dev/null +++ b/SEPythonModule/src/lib/Segmentation.cpp @@ -0,0 +1,63 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Segmentation.h" +#include "Pyston/GIL.h" +#include "SEFramework/Pipeline/SourceGrouping.h" +#include "SEImplementation/Configuration/DetectionFrameConfig.h" +#include "SEImplementation/Segmentation/SegmentationFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include +#include + +namespace SourceXPy { + +namespace py = boost::python; + +using SourceXtractor::DetectionFrameConfig; +using SourceXtractor::DetectionImageFrame; +using SourceXtractor::ProcessSourcesEvent; + +Segmentation::Segmentation(ContextPtr context) : m_context(std::move(context)) { + m_segmentation = m_context->m_segmentation_factory->createSegmentation(); +} + +std::string Segmentation::repr() const { + return "Segmentation"; +} + +void Segmentation::setNextStage(const py::object& callback) { + py::extract> cpp_ifce(callback); + if (cpp_ifce.check()) { + m_next_stage = cpp_ifce(); + } else { + m_next_stage = std::make_shared(callback, m_context); + } + m_segmentation->setNextStage(m_next_stage); +} + +void Segmentation::call() const { + Pyston::SaveThread save_thread; + const auto& detection_config = m_context->m_config_manager->getConfiguration(); + const auto& frames = detection_config.getDetectionFrames(); + boost::for_each(frames, + [this](const std::shared_ptr& frame) { m_segmentation->processFrame(frame); }); + m_next_stage->receiveProcessSignal(ProcessSourcesEvent(std::make_shared())); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/SourceInterface.cpp b/SEPythonModule/src/lib/SourceInterface.cpp new file mode 100644 index 000000000..5e7eba404 --- /dev/null +++ b/SEPythonModule/src/lib/SourceInterface.cpp @@ -0,0 +1,243 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/SourceInterface.h" +#include "SEFramework/Output/OutputRegistry.h" +#include "SEFramework/Property/DetectionFrame.h" +#include "SEFramework/Source/SourceFactory.h" +#include "SEFramework/Source/SourceGroupWithOnDemandPropertiesFactory.h" +#include "SEFramework/Source/SourceInterface.h" +#include "SEImplementation/Configuration/DetectionFrameConfig.h" +#include "SEImplementation/Plugin/GroupInfo/GroupInfo.h" +#include "SEImplementation/Property/SourceId.h" +#include +#include +#include +#include +#include + +namespace SourceXPy { + +namespace py = boost::python; +namespace np = boost::python::numpy; +using Euclid::NdArray::NdArray; +using SourceXtractor::DetectionFrame; +using SourceXtractor::DetectionFrameConfig; +using SourceXtractor::GroupInfo; +using SourceXtractor::PixelCoordinateList; +using SourceXtractor::PropertyId; +using SourceXtractor::SourceId; + +namespace { + +auto logger = Elements::Logging::getLogger("SePy"); + +struct CellToPythonVisitor : public boost::static_visitor { + + template + py::object operator()(const std::vector& vector, + typename std::enable_if::value>::type* = nullptr) const { + auto array = np::zeros(py::make_tuple(vector.size()), np::dtype::get_builtin()); + std::memcpy(array.get_data(), vector.data(), vector.size() * sizeof(T)); + return array; + } + + template + py::object operator()(const NdArray& nd) const { + py::list shape; + boost::for_each(nd.shape(), [&shape](size_t d) { shape.append(d); }); + auto array = np::zeros(py::tuple(shape), np::dtype::get_builtin()); + std::memcpy(array.get_data(), &(*nd.begin()), nd.size() * sizeof(T)); + return array; + } + + py::object operator()(const std::vector& vector) const { + auto array = np::zeros(py::make_tuple(vector.size()), np::dtype::get_builtin()); + size_t i = 0; + for (auto&& v : vector) { + array[i] = v; + ++i; + } + return array; + } + + py::object operator()(const std::string& v) const { + return py::object(v); + } + + template + py::object + operator()(From&& v, + typename std::enable_if::type>::value>::type* = nullptr) const { + return py::object(v); + } +}; + +} // namespace + +std::string DetachedSource::repr() const { + unsigned int source_id = py::extract(m_attributes["source_id"]); + return std::to_string(source_id) + " [DETACHED]"; +} + +py::list DetachedSource::attributes() const { + return m_attributes.keys(); +} + +py::object DetachedSource::attribute(const std::string& key) const { + return m_attributes[key]; +} + +py::object AttachedSource::attribute(const std::string& key) const { + if (!m_source_ptr) { + PyErr_SetString(PyExc_ReferenceError, "Not owned source is gone"); + throw py::error_already_set(); + } + // Trigger computation if needed + try { + auto property_type_index = m_context->m_output_registry->getPropertyForColumn(key); + PropertyId property_id(property_type_index, 0); + m_source_ptr->getProperty(property_id); + } catch (const std::out_of_range&) { + PyErr_SetString(PyExc_AttributeError, key.c_str()); + py::throw_error_already_set(); + } + // Convert + const auto& converter = m_context->m_output_registry->getColumnConverter(key); + return boost::apply_visitor(CellToPythonVisitor(), converter.second(*m_source_ptr)); +} + +DetachedSource AttachedSource::detach() const { + using SourceXtractor::Property; + using SourceXtractor::PropertyId; + + if (!m_source_ptr) { + PyErr_SetString(PyExc_ReferenceError, "Not owned source is gone"); + py::throw_error_already_set(); + } + + DetachedSource detached; + const auto& output_registry = m_context->m_output_registry; + m_source_ptr->visitProperties([&detached, source_ptr = m_source_ptr, + output_registry](const PropertyId& prop_id, const std::shared_ptr&) { + auto columns = output_registry->getColumns(prop_id); + if (columns.empty()) { + logger.debug() << "C++ property serialization not found for " << prop_id.getString(); + } + for (const auto& column : columns) { + if (detached.m_attributes.contains(column)) { + continue; + } + const auto& converter = output_registry->getColumnConverter(column); + detached.m_attributes[column] = boost::apply_visitor(CellToPythonVisitor(), converter.second(*source_ptr)); + } + }); + return detached; +} + +std::string OwnedSource::repr() const { + auto source_id = m_source_ptr->getProperty().getSourceId(); + return std::to_string(source_id) + " [OWNED]"; +} + +std::string EntangledSource::repr() const { + if (!m_source_ptr) { + PyErr_SetString(PyExc_ReferenceError, + "Not owned entangled source is gone. Use its detach method if you need a copy."); + throw py::error_already_set(); + } + auto source_id = m_source_ptr->getProperty().getSourceId(); + return std::to_string(source_id) + " [ENTANGLED]"; +} + +std::string SourceGroup::repr() const { + auto& group_id = m_group->getProperty(); + return std::to_string(group_id.getGroupId()); +} + +size_t SourceGroup::size() const { + return m_group->size(); +} + +py::object SourceGroup::attribute([[maybe_unused]] const std::string& key) const { + return py::object(); +} + +SourceGroup::Iterator SourceGroup::iter() const { + return Iterator{m_group->begin(), m_group->end(), std::make_shared(m_context)}; +} + +std::shared_ptr SourceGroup::Iterator::next() { + if (m_i == m_end) { + // If anyone is still using it, it will fail on access + m_holder->m_source_ptr = nullptr; + PyErr_SetNone(PyExc_StopIteration); + py::throw_error_already_set(); + } + m_holder->m_source_ptr = &m_i->getRef(); + ++m_i; + return m_holder; +} + +namespace { + +PixelCoordinateList PixelCoordinateFromTuple(const boost::python::tuple& tuple) { + const np::ndarray xs = py::extract(tuple[0]); + const np::ndarray ys = py::extract(tuple[1]); + std::vector coordinates(py::len(xs)); + + for (std::size_t i = 0; i < coordinates.size(); ++i) { + coordinates[i].m_x = py::extract(xs[i]); + coordinates[i].m_y = py::extract(ys[i]); + } + + return PixelCoordinateList{std::move(coordinates)}; +} + +} // namespace + +std::shared_ptr OwnedSource::create(const std::shared_ptr& context, int detection_frame_idx, + int detection_id, const boost::python::tuple& pixels) { + const auto& detection_frames = + context->m_config_manager->getConfiguration().getDetectionFrames(); + const auto& detection_frame = detection_frames.at(detection_frame_idx); + + auto source_ptr = context->m_source_factory->createSource(); + source_ptr->setProperty(detection_frame); + source_ptr->setProperty(detection_id); + source_ptr->setProperty(PixelCoordinateFromTuple(pixels)); + + return std::make_shared(context, std::move(source_ptr)); +} + +std::shared_ptr SourceGroup::create(const std::shared_ptr& context, + boost::python::list& sources) { + auto group_ptr = context->m_group_factory->createSourceGroup(); + + auto n_sources = len(sources); + for (ssize_t i = 0; i < n_sources; ++i) { + const auto& source = py::extract>(sources[i]); + // TODO: Do not clone? + group_ptr->addSource(source()->m_owned_source->clone()); + } + + sources.attr("clear")(); + return std::make_shared(SourceGroup{std::move(group_ptr), context}); +} + +} // namespace SourceXPy diff --git a/SEPythonWrapper/CMakeLists.txt b/SEPythonWrapper/CMakeLists.txt new file mode 100644 index 000000000..183a75c15 --- /dev/null +++ b/SEPythonWrapper/CMakeLists.txt @@ -0,0 +1,75 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8.5) + +#=============================================================================== +# Load elements_subdir macro here +# Examples: +# For declaring a project module: +# elements_subdir(ElementsExamples) +#=============================================================================== +elements_subdir(SEPythonWrapper) + +#=============================================================================== +# Load elements_depends_on_subdirs macro here +# For creating a dependency onto an other accessible module +# elements_depends_on_subdirs(ElementsKernel) +#=============================================================================== +elements_depends_on_subdirs(SEImplementation) + +#=============================================================================== +# Add the find_package macro (a pure CMake command) here to locate the +# libraries. +# Examples: +# find_package(CppUnit) +#=============================================================================== + +#=============================================================================== +# Declare the library dependencies here +# Example: +# elements_add_library(ElementsExamples src/Lib/*.cpp +# LINK_LIBRARIES Boost ElementsKernel +# INCLUDE_DIRS Boost ElementsKernel +# PUBLIC_HEADERS ElementsExamples) +#=============================================================================== + +#=============================================================================== +# SEImplementation has a part that needs to be reachable from Python. +#=============================================================================== + +#=============================================================================== +# Declare the executables here +# Example: +# elements_add_executable(ElementsProgramExample src/Program/ProgramExample.cpp +# LINK_LIBRARIES Boost ElementsExamples +# INCLUDE_DIRS Boost ElementsExamples) +#=============================================================================== + +#=============================================================================== +# Declare the Boost tests here +# Example: +# elements_add_unit_test(BoostClassExample tests/src/Boost/ClassExample_test.cpp +# EXECUTABLE BoostClassExample_test +# INCLUDE_DIRS ElementsExamples +# LINK_LIBRARIES ElementsExamples TYPE Boost) +#=============================================================================== + +#=============================================================================== +# Declare the Python programs here +# Examples : +# elements_add_python_program(PythonProgramExample +# ElementsExamples.PythonProgramExample) +#=============================================================================== + +#=============================================================================== +# Use the following macro for python modules, scripts and aux files: +# elements_install_python_modules() +# elements_install_scripts() +#=============================================================================== +elements_install_python_modules() +elements_install_scripts() + +#=============================================================================== +# Add the elements_install_conf_files macro +# Examples: +# elements_install_conf_files() +#=============================================================================== +elements_install_conf_files() diff --git a/SEPythonWrapper/conf/SourceXtractorDemo.conf b/SEPythonWrapper/conf/SourceXtractorDemo.conf new file mode 100644 index 000000000..42bf1fff2 --- /dev/null +++ b/SEPythonWrapper/conf/SourceXtractorDemo.conf @@ -0,0 +1,34 @@ +auto-kron-factor = 2.5 +auto-kron-min-radius = 3.5 +background-cell-size = 64 +smoothing-box-size = 3 +detection-threshold = 1.4953 +segmentation-algorithm = LUTZ +segmentation-use-filtering = true +segmentation-filter = /home/aalvarez/Work/Projects/SourceXtractor-litmus/tests/../data/sim12/default.conv +detection-image-interpolation = 1 +detection-image-interpolation-gap = 5 +use-cleaning = false +cleaning-minimum-area = 10 +detection-minimum-area = 5 +grouping-algorithm = SPLIT +magnitude-zero-point = 32.19 +tile-memory-limit = 512 +tile-size = 256 +model-fitting-iterations = 1000 +thread-count = 4 +partition-multithreshold = true +partition-threshold-count = 32 +partition-minimum-area = 3 +partition-minimum-contrast = 0.005 +psf-fwhm = 3 +psf-pixel-sampling = 1 +weight-use-symmetry = 1 +output-properties = SourceIDs,GroupInfo,PixelCentroid,WorldCentroid,ShapeParameters,IsophotalFlux,SourceFlags +detection-image = /home/aalvarez/Work/Projects/SourceXtractor-litmus/tests/../data/sim12/img/sim12.fits.gz +weight-image = /home/aalvarez/Work/Projects/SourceXtractor-litmus/tests/../data/sim12/img/sim12.weight.fits.gz +weight-type = weight +weight-absolute = True +python-config-file = /home/aalvarez/Work/Projects/SourceXtractor-litmus/tests/../data/sim12/sim12_multi_modelfitting.py +# Custom filter! +snr = 10 diff --git a/SEImplementation/python/sourcextractor/__init__.py b/SEPythonWrapper/python/sourcextractor/__init__.py similarity index 86% rename from SEImplementation/python/sourcextractor/__init__.py rename to SEPythonWrapper/python/sourcextractor/__init__.py index 978074d20..ea8c771e9 100644 --- a/SEImplementation/python/sourcextractor/__init__.py +++ b/SEPythonWrapper/python/sourcextractor/__init__.py @@ -15,5 +15,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from _SourceXtractorPy import Flags +from _SEPythonConfig import Flags +from SOURCEXTRACTORPLUSPLUS_VERSION import SOURCEXTRACTORPLUSPLUS_VERSION_STRING as __version__ diff --git a/SEImplementation/python/sourcextractor/config/__init__.py b/SEPythonWrapper/python/sourcextractor/config/__init__.py similarity index 100% rename from SEImplementation/python/sourcextractor/config/__init__.py rename to SEPythonWrapper/python/sourcextractor/config/__init__.py diff --git a/SEImplementation/python/sourcextractor/config/argv.py b/SEPythonWrapper/python/sourcextractor/config/argv.py similarity index 100% rename from SEImplementation/python/sourcextractor/config/argv.py rename to SEPythonWrapper/python/sourcextractor/config/argv.py diff --git a/SEImplementation/python/sourcextractor/config/compat.py b/SEPythonWrapper/python/sourcextractor/config/compat.py similarity index 99% rename from SEImplementation/python/sourcextractor/config/compat.py rename to SEPythonWrapper/python/sourcextractor/config/compat.py index 292731dd9..fe5dd0c48 100644 --- a/SEImplementation/python/sourcextractor/config/compat.py +++ b/SEPythonWrapper/python/sourcextractor/config/compat.py @@ -17,7 +17,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import sys -import _SourceXtractorPy as cpp +import _SEPythonConfig as cpp from .measurement_config import MeasurementConfig, global_measurement_config from .model_fitting import ModelFitting diff --git a/SEImplementation/python/sourcextractor/config/measurement_config.py b/SEPythonWrapper/python/sourcextractor/config/measurement_config.py similarity index 99% rename from SEImplementation/python/sourcextractor/config/measurement_config.py rename to SEPythonWrapper/python/sourcextractor/config/measurement_config.py index c2756340d..a208b751d 100644 --- a/SEImplementation/python/sourcextractor/config/measurement_config.py +++ b/SEPythonWrapper/python/sourcextractor/config/measurement_config.py @@ -19,7 +19,7 @@ import sys -import _SourceXtractorPy as cpp +import _SEPythonConfig as cpp from .measurement_images import DataCubeSlice, FitsFile, ImageGroup, MeasurementGroup, \ MeasurementImage @@ -88,7 +88,7 @@ def load_fits_image(self, image, psf=None, weight=None, **kwargs): image_file = FitsFile(image) if "image_hdu" in kwargs.keys(): image_hdu_list = [kwargs.pop("image_hdu")] - else: + else: image_hdu_list = image_file.hdu_list # handles the PSFs @@ -99,8 +99,8 @@ def load_fits_image(self, image, psf=None, weight=None, **kwargs): psf_hdu_list = [0] * len(psf_file_list) else: if "psf_hdu" in kwargs.keys(): - psf_hdu_list = [kwargs.pop("psf_hdu")] * len(image_hdu_list) - else: + psf_hdu_list = [kwargs.pop("psf_hdu")] * len(image_hdu_list) + else: psf_hdu_list = range(len(image_hdu_list)) psf_file_list = [psf] * len(image_hdu_list) @@ -117,7 +117,7 @@ def load_fits_image(self, image, psf=None, weight=None, **kwargs): weight_file = FitsFile(weight) if "weight_hdu" in kwargs.keys(): weight_hdu_list = [kwargs.pop("weight_hdu")] * len(image_hdu_list) - else: + else: weight_hdu_list = weight_file.hdu_list weight_file_list = [weight_file] * len(image_hdu_list) diff --git a/SEImplementation/python/sourcextractor/config/measurement_images.py b/SEPythonWrapper/python/sourcextractor/config/measurement_images.py similarity index 99% rename from SEImplementation/python/sourcextractor/config/measurement_images.py rename to SEPythonWrapper/python/sourcextractor/config/measurement_images.py index a5fbbac12..09ca39a76 100644 --- a/SEImplementation/python/sourcextractor/config/measurement_images.py +++ b/SEPythonWrapper/python/sourcextractor/config/measurement_images.py @@ -21,7 +21,7 @@ import re import sys -import _SourceXtractorPy as cpp +import _SEPythonConfig as cpp if sys.version_info.major < 3: from StringIO import StringIO diff --git a/SEImplementation/python/sourcextractor/config/model_fitting.py b/SEPythonWrapper/python/sourcextractor/config/model_fitting.py similarity index 99% rename from SEImplementation/python/sourcextractor/config/model_fitting.py rename to SEPythonWrapper/python/sourcextractor/config/model_fitting.py index 8b8a30413..0cbf9bdd9 100644 --- a/SEImplementation/python/sourcextractor/config/model_fitting.py +++ b/SEPythonWrapper/python/sourcextractor/config/model_fitting.py @@ -22,7 +22,7 @@ import warnings from enum import Enum -import _SourceXtractorPy as cpp +import _SEPythonConfig as cpp try: import pyston diff --git a/SEPythonWrapper/python/sourcextractor/pipeline.py b/SEPythonWrapper/python/sourcextractor/pipeline.py new file mode 100644 index 000000000..f92906a16 --- /dev/null +++ b/SEPythonWrapper/python/sourcextractor/pipeline.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from _SEPythonModule import * + + +class Pipeline: + """ + Wrap a set of pipeline stages and chains them + + :param stages: + List of pipeline stages + """ + + def __init__(self, stages): + if len(stages) < 1: + raise ValueError('Expecting at least one stage') + self.__first = stages[0] + self.__last = stages[-1] + for a, b in zip(stages[:-1], stages[1:]): + a.set_next_stage(b) + + def __call__(self): + """ + Trigger the execution of the pipeline + :return: + The last chain of the pipeline + """ + self.__first() + return self.__last + + +class DefaultPipeline(Pipeline): + """ + Default implementation of the sourcextractor++ pipeline, equivalent to running + the CLI manually + """ + + def __init__(self): + super().__init__([Segmentation(), Partition(), Grouping(), Deblending(), Measurement(), FitsOutput()]) diff --git a/SEPythonWrapper/scripts/CustomGroupingDemo.py b/SEPythonWrapper/scripts/CustomGroupingDemo.py new file mode 100644 index 000000000..03cd0b084 --- /dev/null +++ b/SEPythonWrapper/scripts/CustomGroupingDemo.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import itertools +import logging +import sys +from argparse import ArgumentParser +from configparser import ConfigParser +from typing import Any, Dict, List, Union + +import numpy as np +from sklearn.cluster import AgglomerativeClustering +from sklearn.neighbors import radius_neighbors_graph +from sourcextractor import __version__ as seversion +from sourcextractor import pipeline + +logger = logging.getLogger(__name__) + + +class CustomGrouping: + """ + Grouping based on AgglomerativeClustering. This is just an example of how to perform custom grouping + on sourcextractor++. + + :param context: + sourcextractor++ context + :param kron_scale: + Multiply the Kron radius by this factor to compute the bounding sphere + :param distance_threshold: + Distance threshold for AgglomerativeClustering. 'auto' uses twice the mean radius as an approximation. + """ + + def __init__(self, context: pipeline.Context, kron_scale: float = 2.5, + distance_threshold: Union[float, str] = 'auto'): + self.__context = context + self.__kron_scale = kron_scale + self.__distance_threshold = distance_threshold + self.__to_group = [] + + def set_next_stage(self, next_stage): + """ + Set the following stage, which must receive groups + """ + self.__next = next_stage + + def __filter_sources(self, event: pipeline.ProcessSourcesEvent): + """ + Return a list of sources to flush, and sources to keep + """ + flush, no_flush = [], [] + for source in self.__to_group: + if event.must_process(source): + flush.append(source) + else: + no_flush.append(source) + return flush, no_flush + + def __group(self, sources: List[pipeline.Source]): + """ + Group the given list of sources via AgglomerativeClustering + :return: + A list of Groups + """ + if not sources: + return [] + + logger.info('Grouping %d sources', len(sources)) + + coordinates = np.asarray([(source.pixel_centroid_x, source.pixel_centroid_y) for source in sources]) + radius = 2. * np.asarray([source.kron_radius * self.__kron_scale for source in sources]) + + if self.__distance_threshold == 'auto': + threshold = np.mean(radius) + else: + threshold = self.__distance_threshold + + A = radius_neighbors_graph(coordinates, radius=radius, mode='connectivity', include_self=True) + clustering = AgglomerativeClustering(connectivity=A, n_clusters=None, distance_threshold=threshold) + clusters = clustering.fit_predict(coordinates) + + logger.info('Found %d groups', clusters.max()) + + groups = [[] for _ in range(len(clusters))] + for i, cluster_id in enumerate(clusters): + groups[cluster_id].append(sources[i]) + + groups = list(map(lambda group: pipeline.Group.create(self.__context, group), groups)) + return groups + + def __flush(self, event: pipeline.ProcessSourcesEvent): + """ + Group and flush the sources kept in memory + """ + flusheable, self.__to_group = self.__filter_sources(event) + groups = self.__group(flusheable) + for group in groups: + self.__next(group) + # We need to forward the event + self.__next(event) + + def __call__(self, obj): + """ + Called by segmentation. Use ProcessSourceEvent.must_source to see if stored sources must be + grouped and flushed. + :param obj: + Either a ProcessSourceEvent sent by Segmentation to trigger a flush, or a + segmented source + """ + if isinstance(obj, pipeline.ProcessSourcesEvent): + self.__flush(obj) + else: + self.__to_group.append(obj) + + +def run_sourcextractor(config: Dict[str, Any], output_path: str): + """ + Setup the sourcextractor++ pipeline with a custom grouping stage + """ + config['output-catalog-filename'] = output_path + kron_scale = float(config.get('auto-kron-factor', 2.5)) + distance_threshold = config.pop('grouping-distance-threshold', 'auto') + if distance_threshold != 'auto': + distance_threshold = float(distance_threshold) + context = pipeline.Context(config) + # Generate out custom grouping + grouping = CustomGrouping(context, kron_scale, distance_threshold) + with context: + stages = [pipeline.Segmentation(), pipeline.Partition(), grouping, pipeline.Deblending(), + pipeline.Measurement(), pipeline.FitsOutput()] + pipe = pipeline.Pipeline(stages) + pipe().get() + print(f'Done!') + + +def parse_config_file(path: str) -> Dict[str, Any]: + """ + Parse a sourcextractor++ (like) config file into a dictionary + """ + parser = ConfigParser() + with open(path, 'rt') as fd: + parser.read_file(itertools.chain(['[main]'], fd)) + return {k: v for k, v in parser.items('main')} + + +if __name__ == '__main__': + err_handler = logging.StreamHandler(sys.stderr) + err_handler.setLevel(logging.INFO) + logger.addHandler(err_handler) + logger.setLevel(logging.INFO) + + print(f'Running sourcextractor++ {seversion}') + + parser = ArgumentParser() + parser.add_argument('--output-file', type=str, metavar='FITS', default='output.fits', help='Output file') + parser.add_argument('config_file', type=str, metavar='CONFIGURATION', help='Configuration file') + + args = parser.parse_args() + run_sourcextractor(parse_config_file(args.config_file), args.output_file) diff --git a/SEPythonWrapper/scripts/CustomSegmentationDemo.py b/SEPythonWrapper/scripts/CustomSegmentationDemo.py new file mode 100644 index 000000000..1351c4c1a --- /dev/null +++ b/SEPythonWrapper/scripts/CustomSegmentationDemo.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import itertools +from argparse import ArgumentParser +from configparser import ConfigParser +from typing import Any, Dict, Tuple + +import numpy as np +from astropy.io import fits +from scipy.ndimage import find_objects, label +from sourcextractor import __version__ as seversion +from sourcextractor import pipeline + + +class CustomSegmentation: + """ + Naive segmentation algorithm based on a value / error threshold and scipy.ndimage.label + This is just an example! There is no background subtraction, nor filtering. + Everything is done in memory, so big images will fill the system memory. + + :param context: + sourcextractor++ context + :param detection: + Detection image + :param weight: + Weight image + :param weight_type: + Weight type. Only 'weight', 'variance' and 'rms' are supported. + :param snr: + Threshold for the value / error check + """ + + @staticmethod + def __setup_error(weight_img: np.ndarray, weight_type: str): + """ + Transform the weight image into an error image (sqrt(variance)) + """ + if weight_type == 'weight': + return np.sqrt(np.reciprocal(weight_img)) + elif weight_type == 'variance': + return np.sqrt(weight_img) + elif weight_type == 'rms': + return weight_type + raise ValueError(f'Unsupported weight type {weight_type}') + + def __init__(self, context: pipeline.Context, detection: str, weight: str, weight_type: str, snr: float): + self.__context = context + self.__count = 0 + self.__detection = fits.open(detection)[0].data + self.__std = self.__setup_error(fits.open(weight)[0].data, weight_type) + self.__snr = snr + self.__next = None + + @property + def count(self): + """ + :return: The number of detected sources + """ + return self.__count + + def set_next_stage(self, next_stage): + """ + Set the following stage, which must receive sources + """ + self.__next = next_stage + + @staticmethod + def __get_pixel_coordinates(labeled: np.ndarray, obj: Tuple[slice, slice], idx: int): + """ + Given the labeled image, the object bounding box and its detection ID, return the list + of detected pixels as a tuple (xs, ys) + """ + cutout = labeled[obj] + # numpy axes are 0=Y, 1=X!! + yoff, xoff = np.nonzero(cutout == idx) + return (xoff + obj[1].start).astype(np.int32), (yoff + obj[0].start).astype(np.int32) + + def __call__(self): + """ + Apply the threshold, label the image, and find the objects + """ + # Mask out pixels below the threshold + mask = (self.__detection / self.__std) < self.__snr + self.__detection[mask] = 0. + # Use scipy label + labeled = label(self.__detection > 0)[0] + # Use scipy find_object to find the bounding boxes + detected_objs = find_objects(labeled) + for idx, obj in enumerate(detected_objs, start=1): + self.__count += 1 + pixels = self.__get_pixel_coordinates(labeled, obj, idx) + # We only support a single detection frame in this example + source = pipeline.Source.create(self.__context, detection_frame=0, detection_id=idx, + pixel_coordinates=pixels) + self.__next(source) + self.__next(pipeline.AllFramesDone()) + + +def run_sourcextractor(config: Dict[str, Any], output_path: str): + """ + Setup the sourcextractor++ pipeline with a custom segmentation stage + """ + config['output-catalog-filename'] = output_path + snr = float(config.pop('snr', 5.)) + context = pipeline.Context(config) + # Generate out custom segmentation + segmentation = CustomSegmentation(context, config['detection-image'], config['weight-image'], config['weight-type'], + snr) + with context: + stages = [segmentation, pipeline.Partition(), pipeline.Grouping(), pipeline.Deblending(), + pipeline.Measurement(), pipeline.FitsOutput()] + pipe = pipeline.Pipeline(stages) + pipe().get() + print(f'Done! Found {segmentation.count} sources') + + +def parse_config_file(path: str) -> Dict[str, Any]: + """ + Parse a sourcextractor++ (like) config file into a dictionary + """ + parser = ConfigParser() + with open(path, 'rt') as fd: + parser.read_file(itertools.chain(['[main]'], fd)) + return {k: v for k, v in parser.items('main')} + + +if __name__ == '__main__': + print(f'Running sourcextractor++ {seversion}') + + parser = ArgumentParser() + parser.add_argument('--output-file', type=str, metavar='FITS', default='output.fits', help='Output file') + parser.add_argument('config_file', type=str, metavar='CONFIGURATION', help='Configuration file') + + args = parser.parse_args() + run_sourcextractor(parse_config_file(args.config_file), args.output_file) diff --git a/SEPythonWrapper/scripts/LivePreviewDemo.py b/SEPythonWrapper/scripts/LivePreviewDemo.py new file mode 100644 index 000000000..239d49750 --- /dev/null +++ b/SEPythonWrapper/scripts/LivePreviewDemo.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import itertools +import threading +from argparse import ArgumentParser +from configparser import ConfigParser +from typing import Any, Dict + +import matplotlib.pyplot as plt +from astropy.io import fits +from matplotlib.animation import FuncAnimation +from matplotlib.colors import SymLogNorm +from sourcextractor import __version__ as seversion +from sourcextractor import pipeline + + +class LivePreview: + """ + Display sources on a matplotlib figure as they are being detected + """ + + def __init__(self, image): + self.__next_stage = None + self.__figure, self.__ax = plt.subplots() + self.__height, self.__width = fits.open(image)[0].data.shape + self.__ani = FuncAnimation(self.__figure, self.__update, interval=1000, init_func=self.__init_fig, blit=False) + self.__artists = [] + self.__norm = SymLogNorm(10) + + def set_next_stage(self, next_stage): + self.__next_stage = next_stage + + def __init_fig(self): + self.__ax.set_xlim(0, self.__width) + self.__ax.set_ylim(0, self.__height) + return [] + + def __update(self, frame: int): + artists = self.__artists + self.__artists = [] + return artists + + def __add_stamp(self, source): + stamp = source.detection_filtered_stamp + pos_x, pos_y = source.pixel_centroid_x, source.pixel_centroid_y + left, right = pos_x - stamp.shape[1] // 2, pos_x + stamp.shape[1] // 2 + bottom, top = pos_y - stamp.shape[0] // 2, pos_y + stamp.shape[1] // 2 + self.__artists.append( + self.__ax.imshow(stamp, origin='lower', extent=(left, right, bottom, top), cmap='Greys_r', norm=self.__norm) + ) + + def show(self): + self.__figure.show() + + def stop(self): + self.__ani.event_source.stop() + + def __call__(self, obj): + if isinstance(obj, pipeline.Source): + self.__add_stamp(obj) + elif isinstance(obj, pipeline.Group): + [self.__add_stamp(source) for source in obj] + self.__next_stage(obj) + + +def run_sourcextractor(config: Dict[str, Any], output_path: str): + """ + Setup the sourcextractor++ pipeline + """ + config['output-catalog-filename'] = output_path + live = LivePreview(config['detection-image']) + with pipeline.Context(config): + stages = [pipeline.Segmentation(), pipeline.Partition(), + pipeline.Grouping(), pipeline.Deblending(), live, pipeline.Measurement(), pipeline.FitsOutput()] + pipe = pipeline.Pipeline(stages) + + # UI must run on the main thread, so sourcextractor++ must run on another + sourcex_thread = threading.Thread(target=lambda: pipe().get()) + sourcex_thread.start() + while sourcex_thread.is_alive(): + live.show() + plt.pause(0.05) + sourcex_thread.join() + print(f'Done!') + live.stop() + plt.show(block=True) + + +def parse_config_file(path: str) -> Dict[str, Any]: + """ + Parse a sourcextractor++ (like) config file into a dictionary + """ + parser = ConfigParser() + with open(path, 'rt') as fd: + parser.read_file(itertools.chain(['[main]'], fd)) + return {k: v for k, v in parser.items('main')} + + +if __name__ == '__main__': + print(f'Running sourcextractor++ {seversion}') + + parser = ArgumentParser() + parser.add_argument('--output-file', type=str, metavar='FITS', default='output.fits', help='Output file') + parser.add_argument('config_file', type=str, metavar='CONFIGURATION', help='Configuration file') + + args = parser.parse_args() + run_sourcextractor(parse_config_file(args.config_file), args.output_file) diff --git a/SEPythonWrapper/scripts/SourceXtractorDemo.py b/SEPythonWrapper/scripts/SourceXtractorDemo.py new file mode 100644 index 000000000..49dda932e --- /dev/null +++ b/SEPythonWrapper/scripts/SourceXtractorDemo.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import itertools +import os.path +from argparse import ArgumentParser +from configparser import ConfigParser +from datetime import timedelta +from typing import Any, Dict + +import h5py +import numpy as np +from sourcextractor import __version__ as seversion +from sourcextractor import pipeline + + +class SNRFilter: + """ + Drop sources with a signal-to-noise below the configured limit. + It expect sources, so it must be inserted into the pipeline *before* the partitioning + + :param snr: float + Signal-to-noise ratio cut + """ + + def __init__(self, snr: float): + self.__snr = snr + self.__next = None + self.__dropped = [] + + @property + def dropped(self): + return self.__dropped + + def set_next_stage(self, stage): + self.__next = stage + + def __call__(self, obj): + """ + Apply the SNR filter + """ + if isinstance(obj, pipeline.Source): + if obj.isophotal_flux / obj.isophotal_flux_err < self.__snr: + self.__dropped.append(obj) + return + self.__next(obj) + + +class StoreStamps: + """ + Store the detection stamps into the HDF5 file + + :param hd5: h5py.File + Output HDF5 file + """ + + def __init__(self, hd5: h5py.File): + self.__hd5 = hd5 + self.__next = None + + def set_next_stage(self, stage): + self.__next = stage + + def __store_stamp(self, source): + stamp = source.detection_filtered_stamp + dataset = self.__hd5.create_dataset(f'sources/{source.source_id}', data=stamp) + dataset.attrs.create('CLASS', 'IMAGE', dtype='S6') + dataset.attrs.create('IMAGE_VERSION', '1.2', dtype='S4') + dataset.attrs.create('IMAGE_SUBCLASS', 'IMAGE_GRAYSCALE', dtype='S16') + dataset.attrs.create('IMAGE_MINMAXRANGE', [np.min(stamp), np.max(stamp)]) + + def __call__(self, obj): + """ + Supports being called with a single Source, or with a Group of sources + """ + if isinstance(obj, pipeline.Source): + self.__store_stamp(obj) + elif isinstance(obj, pipeline.Group): + [self.__store_stamp(source) for source in obj] + elif not isinstance(obj, pipeline.ProcessSourcesEvent): + print(f'Unknown {type(obj)}') + self.__next(obj) + + +def run_sourcextractor(config: Dict[str, Any], output_path: str, stamps: bool): + """ + Setup the sourcextractor++ pipeline, run it, and write the output to an HDF5 file + """ + h5 = None + if output_path.endswith('.h5'): + h5 = h5py.File(output_path, 'w') + Output = pipeline.NumpyOutput + else: + Output = pipeline.FitsOutput + config['output-catalog-filename'] = output_path + + timeout = config.pop('timeout', timedelta(days=365)) + snr_filter = SNRFilter(float(config.pop('snr', 5))) + with pipeline.Context(config): + stages = [pipeline.Segmentation(), pipeline.Partition(), snr_filter, pipeline.Grouping(), pipeline.Deblending()] + if stamps and h5 is not None: + stages.append(StoreStamps(h5)) + stages.extend([pipeline.Measurement(), Output()]) + pipe = pipeline.Pipeline(stages) + result = pipe().get(timeout=timeout) + print(f'Dropped {len(snr_filter.dropped)} sources') + + if h5 is not None: + h5.create_dataset(os.path.basename(config['detection-image']), data=result) + h5.close() + print(f'{output_path} created!') + + +def parse_config_file(path: str) -> Dict[str, Any]: + """ + Parse a sourcextractor++ (like) config file into a dictionary + """ + parser = ConfigParser() + with open(path, 'rt') as fd: + parser.read_file(itertools.chain(['[main]'], fd)) + return {k: v for k, v in parser.items('main')} + + +# Entry point +if __name__ == '__main__': + print(f'Running sourcextractor++ {seversion}') + + parser = ArgumentParser() + parser.add_argument('--output-file', type=str, metavar='HDF5', default='output.h5', help='Output file') + parser.add_argument('--with-stamps', action='store_true', default=False, help='Store source stamps') + parser.add_argument('config_file', type=str, metavar='CONFIGURATION', help='Configuration file') + + args = parser.parse_args() + run_sourcextractor(parse_config_file(args.config_file), args.output_file, args.with_stamps) diff --git a/doc/src/conf.py b/doc/src/conf.py index 70c8f2597..74570a709 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -134,7 +134,7 @@ # unit titles (such as .. function::). add_module_names = False -autodoc_mock_imports = ['.measurement_images', '_SourceXtractorPy'] +autodoc_mock_imports = ['.measurement_images', '_SEPythonConfig'] # -- Options for HTML output ---------------------------------------------- @@ -466,7 +466,7 @@ def setup(app): # -- Options for pybtex ---------------------------------------------- bibtex_bibfiles = ["references.bib"] - + from packaging import version as vers from pybtex import __version__ as pybtex_version from pybtex.style.formatting.unsrt import Style as UnsrtStyle, date, pages, toplevel