Graph Neural Networks (GNNs) are a large class of relational models for graph processing. Recent theoretical studies on the expressive power of GNNs have focused on two issues. On the one hand, it has been proven that GNNs are as powerful as the Weisfeiler–Lehman test (1-WL) in their ability to distinguish graphs. Moreover, it has been shown that the equivalence enforced by 1-WL equals unfolding equivalence. On the other hand, GNNs turned out to be universal approximators on graphs modulo the constraints enforced by 1-WL/unfolding equivalence. However, these results only apply to Static Attributed Undirected Homogeneous Graphs (SAUHG) with node attributes. In contrast, real-life applications often involve a much larger variety of graph types. In this paper, we conduct a theoretical analysis of the expressive power of GNNs for two other graph domains that are particularly interesting in practical applications, namely dynamic graphs and SAUGHs with edge attributes. Dynamic graphs are widely used in modern applications; hence, the study of the expressive capability of GNNs in this domain is essential for practical reasons and, in addition, it requires a new analyzing approach due to the difference in the architecture of dynamic GNNs compared to static ones. On the other hand, the examination of SAUHGs is of particular relevance since they act as a standard form for all graph types: it has been shown that all graph types can be transformed without loss of information to SAUHGs with both attributes on nodes and edges. This paper considers generic GNN models and appropriate 1-WL tests for those domains. Then, the known results on the expressive power of GNNs are extended to the mentioned domains: it is proven that GNNs have the same capability as the 1-WL test, the 1-WL equivalence equals unfolding equivalence and that GNNs are universal approximators modulo 1-WL/unfolding equivalence. Moreover, the proof of the approximation capability is mostly constructive and allows us to deduce hints on the architecture of GNNs that can achieve the desired approximation.

@article{beddarwiesing2024weisfeiler, author = {Beddar-Wiesing, Silvia and D'Inverno, Alessio and Graziani, Caterina and Lachi, Veronica and Moallemy-Oureh, Alice and Scarselli, Franco and Thomas, Josephine}, title = {Weisfeiler–Lehman goes dynamic: An analysis of the expressive power of Graph Neural Networks for attributed and dynamic graphs}, journal = {Neural Networks}, publisher = {Elsevier}, year = {2024}, volume = {173}, pages = {106213}, url = {https://www.sciencedirect.com/science/article/pii/S0893608024001370}, doi = {10.1016/j.neunet.2024.106213}, keywords = {imported, itegpub, isac-www, Graph_Neural_Network, Dynamic_graphs, GNN_expressivity, Unfolding_Trees, Weisfeiler–Lehman, test}, abstract = {Graph Neural Networks (GNNs) are a large class of relational models for graph processing. Recent theoretical studies on the expressive power of GNNs have focused on two issues. On the one hand, it has been proven that GNNs are as powerful as the Weisfeiler–Lehman test (1-WL) in their ability to distinguish graphs. Moreover, it has been shown that the equivalence enforced by 1-WL equals unfolding equivalence. On the other hand, GNNs turned out to be universal approximators on graphs modulo the constraints enforced by 1-WL/unfolding equivalence. However, these results only apply to Static Attributed Undirected Homogeneous Graphs (SAUHG) with node attributes. In contrast, real-life applications often involve a much larger variety of graph types. In this paper, we conduct a theoretical analysis of the expressive power of GNNs for two other graph domains that are particularly interesting in practical applications, namely dynamic graphs and SAUGHs with edge attributes. Dynamic graphs are widely used in modern applications; hence, the study of the expressive capability of GNNs in this domain is essential for practical reasons and, in addition, it requires a new analyzing approach due to the difference in the architecture of dynamic GNNs compared to static ones. On the other hand, the examination of SAUHGs is of particular relevance since they act as a standard form for all graph types: it has been shown that all graph types can be transformed without loss of information to SAUHGs with both attributes on nodes and edges. This paper considers generic GNN models and appropriate 1-WL tests for those domains. Then, the known results on the expressive power of GNNs are extended to the mentioned domains: it is proven that GNNs have the same capability as the 1-WL test, the 1-WL equivalence equals unfolding equivalence and that GNNs are universal approximators modulo 1-WL/unfolding equivalence. Moreover, the proof of the approximation capability is mostly constructive and allows us to deduce hints on the architecture of GNNs that can achieve the desired approximation.} }

%0 = article %A = Beddar-Wiesing, Silvia and D'Inverno, Alessio and Graziani, Caterina and Lachi, Veronica and Moallemy-Oureh, Alice and Scarselli, Franco and Thomas, Josephine %D = 2024 %I = Elsevier %T = Weisfeiler–Lehman goes dynamic: An analysis of the expressive power of Graph Neural Networks for attributed and dynamic graphs %U = https://www.sciencedirect.com/science/article/pii/S0893608024001370

This paper presents a methodological framework for training, self-optimising, and self-organising surrogate models to approximate and speed up multiobjective optimisation of technical systems based on multiphysics simulations. At the hand of two real-world datasets, we illustrate that surrogate models can be trained on relatively small amounts of data to approximate the underlying simulations accurately. Including explainable AI techniques allow for highlighting feature relevancy or dependencies and supporting the possible extension of the used datasets. One of the datasets was created for this paper and is made publicly available for the broader scientific community. Extensive experiments combine four machine learning and deep learning algorithms with an evolutionary optimisation algorithm. The performance of the combined training and optimisation pipeline is evaluated by verifying the generated Pareto-optimal results using the ground truth simulations. The results from our pipeline and a comprehensive evaluation strategy show the potential for efficiently acquiring solution candidates in multiobjective optimisation tasks by reducing the number of simulations and conserving a higher prediction accuracy, i.e., with a MAPE score under 5% for one of the presented use cases.

@article{botache2024enhancing, author = {Botache, Diego and Decke, Jens and Ripken, Winfried and Dornipati, Abhinay and Götz-Hahn, Franz and Ayeb, Mohamed and Sick, Bernhard}, title = {Enhancing Multi-Objective Optimization through Machine Learning-Supported Multiphysics Simulation}, journal = {arXiv e-prints}, year = {2024}, pages = {arXiv:2309.13179v2}, url = {https://arxiv.org/abs/2309.13179}, keywords = {imported, itegpub, isac-www, Electric_Motors, Multiobjective_Optimisation, Surrogate-Modelling, Deep-Learning, Explainable_Artificial_Intelligence}, abstract = {This paper presents a methodological framework for training, self-optimising, and self-organising surrogate models to approximate and speed up multiobjective optimisation of technical systems based on multiphysics simulations. At the hand of two real-world datasets, we illustrate that surrogate models can be trained on relatively small amounts of data to approximate the underlying simulations accurately. Including explainable AI techniques allow for highlighting feature relevancy or dependencies and supporting the possible extension of the used datasets. One of the datasets was created for this paper and is made publicly available for the broader scientific community. Extensive experiments combine four machine learning and deep learning algorithms with an evolutionary optimisation algorithm. The performance of the combined training and optimisation pipeline is evaluated by verifying the generated Pareto-optimal results using the ground truth simulations. The results from our pipeline and a comprehensive evaluation strategy show the potential for efficiently acquiring solution candidates in multiobjective optimisation tasks by reducing the number of simulations and conserving a higher prediction accuracy, i.e., with a MAPE score under 5% for one of the presented use cases.} }

%0 = article %A = Botache, Diego and Decke, Jens and Ripken, Winfried and Dornipati, Abhinay and Götz-Hahn, Franz and Ayeb, Mohamed and Sick, Bernhard %D = 2024 %T = Enhancing Multi-Objective Optimization through Machine Learning-Supported Multiphysics Simulation %U = https://arxiv.org/abs/2309.13179

This article presents the Sorting Composite Quantile Regression Neural Network (SCQRNN), an advanced quantile regression model designed to prevent quantile crossing and enhance computational efficiency. Integrating ad hoc sorting in training, the SCQRNN ensures non-intersecting quantiles, boosting model reliability and interpretability. We demonstrate that the SCQRNN not only prevents quantile crossing and reduces computational complexity but also achieves faster convergence than traditional models. This advancement meets the requirements of high-performance computing for sustainable, accurate computation. In organic computing, the SCQRNN enhances self-aware systems with predictive uncertainties, enriching applications across finance, meteorology, climate science, and engineering.

@inproceedings{decke2024efficient, author = {Decke, Jens and Jenß, Arne and Sick, Bernhard and Gruhl, Christian}, title = {An Efficient Multi Quantile Regression Network with Ad Hoc Prevention of Quantile Crossing}, booktitle = {International Conference on Architecture of Computing Systems (ARCS)}, publisher = {Springer}, year = {2024}, note = {(accepted)}, keywords = {imported, itegpub, isac-www, QuantileRegression, QuantileCrossing, OrganicComputing, Self-Awareness, DifferentiableSorting}, abstract = {This article presents the Sorting Composite Quantile Regression Neural Network (SCQRNN), an advanced quantile regression model designed to prevent quantile crossing and enhance computational efficiency. Integrating ad hoc sorting in training, the SCQRNN ensures non-intersecting quantiles, boosting model reliability and interpretability. We demonstrate that the SCQRNN not only prevents quantile crossing and reduces computational complexity but also achieves faster convergence than traditional models. This advancement meets the requirements of high-performance computing for sustainable, accurate computation. In organic computing, the SCQRNN enhances self-aware systems with predictive uncertainties, enriching applications across finance, meteorology, climate science, and engineering.} }

%0 = inproceedings %A = Decke, Jens and Jenß, Arne and Sick, Bernhard and Gruhl, Christian %B = International Conference on Architecture of Computing Systems (ARCS) %D = 2024 %I = Springer %T = An Efficient Multi Quantile Regression Network with Ad Hoc Prevention of Quantile Crossing

This article investigates the application of computer vision and graph-based models in solving mesh-based partial differential equations within high-performance computing environments. Focusing on structured, graded structured, and unstructured meshes, the study compares the performance and computational efficiency of three computer vision-based models against three graph-based models across three datasets. The research aims to identify the most suitable models for different mesh topographies, particularly highlighting the exploration of graded meshes, a less studied area. Results demonstrate that computer vision-based models, notably U-Net, outperform the graph models in prediction performance and efficiency in two (structured and graded) out of three mesh topographies. The study also reveals the unexpected effectiveness of computer vision-based models in handling unstructured meshes, suggesting a potential shift in methodological approaches for data-driven partial differential equation learning. The article underscores deep learning as a viable and potentially sustainable way to enhance traditional high-performance computing methods, advocating for informed model selection based on the topography of the mesh.

@inproceedings{decke2024structured, author = {Decke, Jens and Wünsch, Olaf and Sick, Bernhard and Gruhl, Christian}, title = {From Structured to Unstructured: A Comparative Analysis of Computer Vision and Graph Models in solving Mesh-based PDEs}, booktitle = {International Conference on Architecture of Computing Systems (ARCS)}, publisher = {Springer}, year = {2024}, note = {(accepted)}, keywords = {imported, itegpub, isac-www, Organic, Computing, Self-Optimization, DeepLearning, PartialDifferentialEquation, SurrogateModel, MeshTopographies}, abstract = {This article investigates the application of computer vision and graph-based models in solving mesh-based partial differential equations within high-performance computing environments. Focusing on structured, graded structured, and unstructured meshes, the study compares the performance and computational efficiency of three computer vision-based models against three graph-based models across three datasets. The research aims to identify the most suitable models for different mesh topographies, particularly highlighting the exploration of graded meshes, a less studied area. Results demonstrate that computer vision-based models, notably U-Net, outperform the graph models in prediction performance and efficiency in two (structured and graded) out of three mesh topographies. The study also reveals the unexpected effectiveness of computer vision-based models in handling unstructured meshes, suggesting a potential shift in methodological approaches for data-driven partial differential equation learning. The article underscores deep learning as a viable and potentially sustainable way to enhance traditional high-performance computing methods, advocating for informed model selection based on the topography of the mesh.} }

%0 = inproceedings %A = Decke, Jens and Wünsch, Olaf and Sick, Bernhard and Gruhl, Christian %B = International Conference on Architecture of Computing Systems (ARCS) %D = 2024 %I = Springer %T = From Structured to Unstructured: A Comparative Analysis of Computer Vision and Graph Models in solving Mesh-based PDEs

The operating environment of a highly automated vehicle is subject to change, e.g., weather, illumination, or the scenario containing different objects and other participants in which the highly automated vehicle has to navigate its passengers safely. These situations must be considered when developing and validating highly automated driving functions. This already poses a problem for training and evaluating deep learning models because without the costly labeling of thousands of recordings, not knowing whether the data contains relevant, interesting data for further model training, it is a guess under which conditions and situations the model performs poorly. For this purpose, we present corner case criteria based on the predictive uncertainty. With our corner case criteria, we are able to detect uncertainty-based corner cases of an object instance segmentation model without relying on ground truth (GT) data. We evaluated each corner case criterion using the COCO and the NuImages dataset to analyze the potential of our approach. We also provide a corner case decision function that allows us to distinguish each object into True Positive (TP), localization and/or classification corner case, or False Positive (FP). We also present our first results of an iterative training cycle that outperforms the baseline and where the data added to the training dataset is selected based on the corner case decision function.

@article{heidecker2024criteria, author = {Heidecker, Florian and El-Khateeb, Ahmad and Bieshaar, Maarten and Sick, Bernhard}, title = {Criteria for Uncertainty-based Corner Cases Detection in Instance Segmentation}, journal = {arXiv e-prints}, year = {2024}, pages = {arXiv:2404.11266}, url = {https://arxiv.org/abs/2404.11266}, keywords = {imported, itegpub, isac-www}, abstract = {The operating environment of a highly automated vehicle is subject to change, e.g., weather, illumination, or the scenario containing different objects and other participants in which the highly automated vehicle has to navigate its passengers safely. These situations must be considered when developing and validating highly automated driving functions. This already poses a problem for training and evaluating deep learning models because without the costly labeling of thousands of recordings, not knowing whether the data contains relevant, interesting data for further model training, it is a guess under which conditions and situations the model performs poorly. For this purpose, we present corner case criteria based on the predictive uncertainty. With our corner case criteria, we are able to detect uncertainty-based corner cases of an object instance segmentation model without relying on ground truth (GT) data. We evaluated each corner case criterion using the COCO and the NuImages dataset to analyze the potential of our approach. We also provide a corner case decision function that allows us to distinguish each object into True Positive (TP), localization and/or classification corner case, or False Positive (FP). We also present our first results of an iterative training cycle that outperforms the baseline and where the data added to the training dataset is selected based on the corner case decision function.} }

%0 = article %A = Heidecker, Florian and El-Khateeb, Ahmad and Bieshaar, Maarten and Sick, Bernhard %D = 2024 %T = Criteria for Uncertainty-based Corner Cases Detection in Instance Segmentation %U = https://arxiv.org/abs/2404.11266

@book{tomforde2024organic,, title = {Organic Computing -- Doctoral Dissertation Colloquium 2023}, editor = {Tomforde, Sven and Krupitzer, Christian}, series = {Intelligent Embedded Systems}, publisher = {kassel university press}, year = {2024}, volume = {26}, doi = {10.17170/kobra-202402269661}, keywords = {imported, itegpub, isac-www} }

%0 = book %D = 2024 %I = kassel university press %T = Organic Computing -- Doctoral Dissertation Colloquium 2023

The increasing environmental pollution and rising global energy demands require the rapid development of novel and sustainable traction machines from urban transport up to large-scale transport vehicles. The optimisation of the development cycle of electrical traction machines plays a vital role in sustainable mobility. Still, developers have to face the difficulty of analysing time-consuming multiphysics simulations to assess the performance of many prototype variants. Accelerating this process involves implementing machine learning-supported strategies that can effectively acquire results and predictions about the performance and efficiency of prototypes in an iterative optimisation process and detect possible issues during the experimental procedure. The following research project streamlines the development cycle by identifying and presenting multiple strategies for applying and evaluating machine learning and deep learning techniques in different stages and areas in the development cycle including self-optimisation and self-adaptation capabilities. By implementing these strategies, the optimisation process can be further improved and become more efficient and robust, reducing the time and cost required for testing and validation. Therefore, this proposal addresses two application areas. The first area consists of the supported machine learning motor performance prediction at the topology design and optimisation stage of structural components. The second area corresponds to the machine learning-supported experimental evaluation process of prototypes at desired test benches.

@incollection{botache2023machine, author = {Botache, Diego}, title = {Machine Learning Supported Optimisation and Experimental Evaluation of Electrical Motors for Small Urban Passenger Vehicles}, editor = {Tomforde, Sven and Krupitzer, Christian}, booktitle = {Organic Computing -- Doctoral Dissertation Colloquium 2023}, publisher = {kassel university press}, year = {2023}, pages = {14--26}, keywords = {imported, itegpub, isac-www, electrical_traction_machines, sustainable_mobility, data-driven_models, machine_learning, multi-objective_optimisation, experimental_evaluation, deep_learning_techniques, test_benches}, abstract = {The increasing environmental pollution and rising global energy demands require the rapid development of novel and sustainable traction machines from urban transport up to large-scale transport vehicles. The optimisation of the development cycle of electrical traction machines plays a vital role in sustainable mobility. Still, developers have to face the difficulty of analysing time-consuming multiphysics simulations to assess the performance of many prototype variants. Accelerating this process involves implementing machine learning-supported strategies that can effectively acquire results and predictions about the performance and efficiency of prototypes in an iterative optimisation process and detect possible issues during the experimental procedure. The following research project streamlines the development cycle by identifying and presenting multiple strategies for applying and evaluating machine learning and deep learning techniques in different stages and areas in the development cycle including self-optimisation and self-adaptation capabilities. By implementing these strategies, the optimisation process can be further improved and become more efficient and robust, reducing the time and cost required for testing and validation. Therefore, this proposal addresses two application areas. The first area consists of the supported machine learning motor performance prediction at the topology design and optimisation stage of structural components. The second area corresponds to the machine learning-supported experimental evaluation process of prototypes at desired test benches. } }

%0 = incollection %A = Botache, Diego %B = Organic Computing -- Doctoral Dissertation Colloquium 2023 %D = 2023 %I = kassel university press %T = Machine Learning Supported Optimisation and Experimental Evaluation of Electrical Motors for Small Urban Passenger Vehicles

In the field of design optimization, numerical equation solvers,
mmonly used for finite element analysis and computational fluid dy-
mics, impose significant computational demands. The iterative nature
design optimization, involving numerous simulations, magnifies re-
urce requirements in terms of time, energy, costs, and greenhouse gas
issions. Accelerating this process has been a long-standing research
allenge, driven by the potential for resource savings and the ability
tackle increasingly complex problems. Recently, organic computing
thods have gained prominence as promising approaches to address
is challenge. This study aims to bridge the gap by integrating tech-
ques from deep learning, active learning, and generative learning into
e field of design optimization. The goal is to accelerate the design op-
mization process while addressing specific challenges such as exploring
rge and complex design spaces and evaluating the advantages and con-
raints of these methods. This research has the potential to significantly
pact the industrial use of design optimization by providing faster and
re efficient tools.

@incollection{decke2023examination, author = {Decke, Jens}, title = {An Examination of Organic Computing Strategies in Design Optimization}, editor = {Tomforde, Sven and Krupitzer, Christian}, booktitle = {Organic Computing -- Doctoral Dissertation Colloquium 2023}, publisher = {kassel university press}, year = {2023}, pages = {41--54}, keywords = {imported, itegpub, isac-www, DesignOptimization, DeepLearning, ActiveLearning, GenerativeLearning}, abstract = { In the field of design optimization, numerical equation solvers,
mmonly used for finite element analysis and computational fluid dy-
mics, impose significant computational demands. The iterative nature
design optimization, involving numerous simulations, magnifies re-
urce requirements in terms of time, energy, costs, and greenhouse gas
issions. Accelerating this process has been a long-standing research
allenge, driven by the potential for resource savings and the ability
tackle increasingly complex problems. Recently, organic computing
thods have gained prominence as promising approaches to address
is challenge. This study aims to bridge the gap by integrating tech-
ques from deep learning, active learning, and generative learning into
e field of design optimization. The goal is to accelerate the design op-
mization process while addressing specific challenges such as exploring
rge and complex design spaces and evaluating the advantages and con-
raints of these methods. This research has the potential to significantly
pact the industrial use of design optimization by providing faster and
re efficient tools.} }

%0 = incollection %A = Decke, Jens %B = Organic Computing -- Doctoral Dissertation Colloquium 2023 %D = 2023 %I = kassel university press %T = An Examination of Organic Computing Strategies in Design Optimization

In this work, we apply deep active learning to the field of design optimization to reduce the number of computationally expensive numerical simulations widely used in industry and engineering.
are interested in optimizing the design of structural components, where a set of parameters describes the shape. If we can predict the performance based on these parameters and consider only the promising candidates for simulation, there is an enormous potential for saving computing power.
present two query strategies for self-optimization to reduce the computational cost in multi-objective design optimization problems. Our proposed methodology provides an intuitive approach that is easy to apply, offers significant improvements over random sampling, and circumvents the need for uncertainty estimation.
evaluate our strategies on a large dataset from the domain of fluid dynamics and introduce two new evaluation metrics to determine the model's performance.
ndings from our evaluation highlights the effectiveness of our query strategies in accelerating design optimization.
rthermore, the introduced method is easily transferable to other self-optimization problems in industry and engineering.

@inproceedings{decke2023dado, author = {Decke, Jens and Gruhl, Christian and Rauch, Lukas and Sick, Bernhard}, title = {DADO – Low-Cost Query Strategies for Deep Active Design Optimization}, booktitle = {International Conference on Machine Learning and Applications (ICMLA)}, publisher = {IEEE}, year = {2023}, pages = {1611--1618}, doi = {10.1109/ICMLA58977.2023.00244}, keywords = {imported, itegpub, isac-www, Self-Optimization, Self-Supervised-Learning, Design-Optimization, Active-Learning, Numerical-Simulation}, abstract = {In this work, we apply deep active learning to the field of design optimization to reduce the number of computationally expensive numerical simulations widely used in industry and engineering.
are interested in optimizing the design of structural components, where a set of parameters describes the shape. If we can predict the performance based on these parameters and consider only the promising candidates for simulation, there is an enormous potential for saving computing power.
present two query strategies for self-optimization to reduce the computational cost in multi-objective design optimization problems. Our proposed methodology provides an intuitive approach that is easy to apply, offers significant improvements over random sampling, and circumvents the need for uncertainty estimation.
evaluate our strategies on a large dataset from the domain of fluid dynamics and introduce two new evaluation metrics to determine the model's performance.
ndings from our evaluation highlights the effectiveness of our query strategies in accelerating design optimization.
rthermore, the introduced method is easily transferable to other self-optimization problems in industry and engineering.} }

%0 = inproceedings %A = Decke, Jens and Gruhl, Christian and Rauch, Lukas and Sick, Bernhard %B = International Conference on Machine Learning and Applications (ICMLA) %D = 2023 %I = IEEE %T = DADO – Low-Cost Query Strategies for Deep Active Design Optimization

Context information provided along with a dataset can be very helpful for solving a problem because the additional knowledge is already available and does not need to be extracted. Moreover, the context indicates how diverse a dataset is, i.e., how many samples per context category are available to train and test machine learning (ML) models. In this article, we present context annotations for the BDD100k image dataset. The annotations comprise, for instance, information about daytime, road condition (dry/wet), and dirt on the windshield. Sometimes, no or only little data are available for unique or rare combinations of these context attributes. However, data that matches these context conditions is crucial when discussing corner cases: Firstly, most ML models, e.g., object detectors, are not trained on such data, which leads to the assumption that they will perform poorly in those situations. Secondly, data containing corner cases are required for validating ML models. With this in mind, separate ML models dedicated to context detection are useful for expanding the training set with additional data of special interest, such as corner cases.

@inproceedings{heidecker2023context, author = {Heidecker, Florian and Susetzky, Tobias and Fuchs, Erich and Sick, Bernhard}, title = {Context Information for Corner Case Detection in Highly Automated Driving}, booktitle = {IEEE International Conference on Intelligent Transportation Systems (ITSC)}, publisher = {IEEE}, year = {2023}, pages = {1522--1529}, doi = {10.1109/ITSC57777.2023.10422414}, keywords = {imported, itegpub, isac-www}, abstract = {Context information provided along with a dataset can be very helpful for solving a problem because the additional knowledge is already available and does not need to be extracted. Moreover, the context indicates how diverse a dataset is, i.e., how many samples per context category are available to train and test machine learning (ML) models. In this article, we present context annotations for the BDD100k image dataset. The annotations comprise, for instance, information about daytime, road condition (dry/wet), and dirt on the windshield. Sometimes, no or only little data are available for unique or rare combinations of these context attributes. However, data that matches these context conditions is crucial when discussing corner cases: Firstly, most ML models, e.g., object detectors, are not trained on such data, which leads to the assumption that they will perform poorly in those situations. Secondly, data containing corner cases are required for validating ML models. With this in mind, separate ML models dedicated to context detection are useful for expanding the training set with additional data of special interest, such as corner cases.} }

%0 = inproceedings %A = Heidecker, Florian and Susetzky, Tobias and Fuchs, Erich and Sick, Bernhard %B = IEEE International Conference on Intelligent Transportation Systems (ITSC) %D = 2023 %I = IEEE %T = Context Information for Corner Case Detection in Highly Automated Driving

RUL prediction plays a crucial role in the health management of industrial systems. Given the increasing complexity of systems, data-driven predictive models have attracted significant research interest. Upon reviewing the existing literature, it appears that many studies either do not fully integrate both spatial and temporal features or employ only a single attention mechanism. Furthermore, there seems to be inconsistency in the choice of data normalization methods, particularly concerning operating conditions, which might influence predictive performance. To bridge these observations, this study presents the Spatio-Temporal Attention Graph Neural Network. Our model combines graph neural networks and temporal convolutional neural networks for spatial and temporal feature extraction, respectively. The cascade of these extractors, combined with multihead attention mechanisms for both spatio-temporal dimensions, aims to improve predictive precision and refine model explainability. Comprehensive experiments were conducted on the CMAPSS dataset to evaluate the impact of unified versus clustering normalization. The findings suggest that our model performs state-of-the-art results using only the unified normalization. Additionally, when dealing with datasets with multiple operating conditions, cluster normalization enhances the performance of our proposed model by up to 27%.

@inproceedings{huang2023spatio, author = {Huang, Zhixin and He, Yujiang and Sick, Bernhard}, title = {Spatio-Temporal Attention Graph Neural Network for Remaining Useful Life Prediction}, booktitle = {Computational Science and Computational Intelligence (CSCI)}, publisher = {IEEE}, year = {2023}, note = {(accepted)}, keywords = {imported, itegpub, isac-www, Spatio-Temporal_Attention, Remaining_Useful_Life, Graph_Neural_Network, RUL_Prediction, Clustering_Normalization}, abstract = {RUL prediction plays a crucial role in the health management of industrial systems. Given the increasing complexity of systems, data-driven predictive models have attracted significant research interest. Upon reviewing the existing literature, it appears that many studies either do not fully integrate both spatial and temporal features or employ only a single attention mechanism. Furthermore, there seems to be inconsistency in the choice of data normalization methods, particularly concerning operating conditions, which might influence predictive performance. To bridge these observations, this study presents the Spatio-Temporal Attention Graph Neural Network. Our model combines graph neural networks and temporal convolutional neural networks for spatial and temporal feature extraction, respectively. The cascade of these extractors, combined with multihead attention mechanisms for both spatio-temporal dimensions, aims to improve predictive precision and refine model explainability. Comprehensive experiments were conducted on the CMAPSS dataset to evaluate the impact of unified versus clustering normalization. The findings suggest that our model performs state-of-the-art results using only the unified normalization. Additionally, when dealing with datasets with multiple operating conditions, cluster normalization enhances the performance of our proposed model by up to 27%.} }

%0 = inproceedings %A = Huang, Zhixin and He, Yujiang and Sick, Bernhard %B = Computational Science and Computational Intelligence (CSCI) %D = 2023 %I = IEEE %T = Spatio-Temporal Attention Graph Neural Network for Remaining Useful Life Prediction

In the domain of graph neural networks (GNNs), pooling operators are fundamental to reduce the size of the graph by simplifying graph structures and vertex features. Recent advances have shown that well-designed pooling operators, coupled with message-passing layers, can endow hierarchical GNNs with an expressive power regarding the graph isomorphism test that is equal to the Weisfeiler-Leman test. However, the ability of hierarchical GNNs to increase expressive power by utilizing graph coarsening was not yet explored. This results in uncertainties about the benefits of pooling operators and a lack of sufficient properties to guide their design. In this work, we identify conditions for pooling operators to generate WL-distinguishable coarsened graphs from originally WL-indistinguishable but non-isomorphic graphs. Our conditions are versatile and can be tailored to specific tasks and data characteristics, offering a promising avenue for further research.

@inproceedings{lachi2023graph, author = {Lachi, Veronica and Moallemy-Oureh, Alice and Roth, Andreas and Welke, Pascal}, title = {Graph Pooling Provably Improves Expressivity}, booktitle = {Workshop on New Frontiers in Graph Learning, NeurIPS}, year = {2023}, pages = {1--7}, url = {https://openreview.net/forum?id=lR5NYB9zrv}, keywords = {imported, itegpub, isac-www}, abstract = {In the domain of graph neural networks (GNNs), pooling operators are fundamental to reduce the size of the graph by simplifying graph structures and vertex features. Recent advances have shown that well-designed pooling operators, coupled with message-passing layers, can endow hierarchical GNNs with an expressive power regarding the graph isomorphism test that is equal to the Weisfeiler-Leman test. However, the ability of hierarchical GNNs to increase expressive power by utilizing graph coarsening was not yet explored. This results in uncertainties about the benefits of pooling operators and a lack of sufficient properties to guide their design. In this work, we identify conditions for pooling operators to generate WL-distinguishable coarsened graphs from originally WL-indistinguishable but non-isomorphic graphs. Our conditions are versatile and can be tailored to specific tasks and data characteristics, offering a promising avenue for further research.} }

%0 = inproceedings %A = Lachi, Veronica and Moallemy-Oureh, Alice and Roth, Andreas and Welke, Pascal %B = Workshop on New Frontiers in Graph Learning, NeurIPS %D = 2023 %T = Graph Pooling Provably Improves Expressivity %U = https://openreview.net/forum?id=lR5NYB9zrv

Often, producing large labelled datasets for supervised machine learning is difficult and expensive.
cases where the expensive part is due to labelling and obtaining ground truth, it is often comparably easy to acquire large datasets containing unlabelled data points. For reproducible measurements, it is possible to record information on multiple data points being from the same reproducible measurement series, which should thus have an equal but unknown ground truth.
this article, we propose a method to incorporate a dataset of such unlabelled data points for which some data points are known to be equal in end-to-end training of otherwise labelled data.
show that, with the example of predicting the carotenoid concentration in human skin from optical multiple spatially resolved reflection spectroscopy data, the proposed method is capable of reducing the required number of labelled data points to achieve the same prediction accuracy for different model architectures. In addition, we show that the proposed method is capable of reducing the negative impact of noisy data when performing a repeated measurement of the same sample.

@inproceedings{magnussen2023leveraging, author = {Magnussen, Birk Martin and Stern, Claudius and Sick, Bernhard}, title = {Leveraging Repeated Unlabelled Noisy Measurements to Augment Supervised Learning}, booktitle = {International Conference on Computational Intelligence and Intelligent Systems (CIIS)}, publisher = {ACM}, year = {2023}, pages = {1--6}, url = {https://dl.acm.org/doi/10.1145/3638209.3638210}, doi = {10.1145/3638209.3638210}, keywords = {imported, itegpub, isac-www, noisy_data, inhomogeneous_labels, neural_networks, semi-supervised_learning, reflection_spectroscopy}, abstract = {Often, producing large labelled datasets for supervised machine learning is difficult and expensive.
cases where the expensive part is due to labelling and obtaining ground truth, it is often comparably easy to acquire large datasets containing unlabelled data points. For reproducible measurements, it is possible to record information on multiple data points being from the same reproducible measurement series, which should thus have an equal but unknown ground truth.
this article, we propose a method to incorporate a dataset of such unlabelled data points for which some data points are known to be equal in end-to-end training of otherwise labelled data.
show that, with the example of predicting the carotenoid concentration in human skin from optical multiple spatially resolved reflection spectroscopy data, the proposed method is capable of reducing the required number of labelled data points to achieve the same prediction accuracy for different model architectures. In addition, we show that the proposed method is capable of reducing the negative impact of noisy data when performing a repeated measurement of the same sample. } }

%0 = inproceedings %A = Magnussen, Birk Martin and Stern, Claudius and Sick, Bernhard %B = International Conference on Computational Intelligence and Intelligent Systems (CIIS) %D = 2023 %I = ACM %T = Leveraging Repeated Unlabelled Noisy Measurements to Augment Supervised Learning %U = https://dl.acm.org/doi/10.1145/3638209.3638210

Spatio-Temporal Point Processes (STPPs) have recently become increasingly interesting for learning dynamic graph data since many scientific fields, ranging from mathematics, biology, social sciences, and physics to computer science, are naturally related and dynamic. While training Recurrent Neural Networks and solving PDEs for representing temporal data is expensive, TPPs were a good alternative. The drawback is that constructing an appropriate TPP for modeling temporal data requires the assumption of a particular temporal behavior of the data. To overcome this problem, Neural TPPs have been developed that enable learning of the parameters of the TPP. However, the research is relatively young for modeling dynamic graphs, and only a few TPPs have been proposed to handle edge-dynamic graphs. To allow for learning on a fully dynamic graph, we propose the first Marked Neural Spatio-Temporal Point Process (MNSTPP) that leverages a Dynamic Graph Neural Network to learn Spatio-TPPs to model and predict any event in a graph stream.
addition, our model can be updated efficiently by considering single events for local retraining.

@inproceedings{moallemyoureh2023marked, author = {Moallemy-Oureh, Alice and Beddar-Wiesing, Silvia and Nather, Rüdiger and Thomas, Josephine}, title = {Marked Neural Spatio-Temporal Point Process Involving a Dynamic Graph Neural Network}, booktitle = {Workshop on Temporal Graph Learning (TGL), NeurIPS}, year = {2023}, pages = {1--7}, url = {https://openreview.net/forum?id=QJx3Cmddsy}, keywords = {imported, itegpub, isac-www}, abstract = {Spatio-Temporal Point Processes (STPPs) have recently become increasingly interesting for learning dynamic graph data since many scientific fields, ranging from mathematics, biology, social sciences, and physics to computer science, are naturally related and dynamic. While training Recurrent Neural Networks and solving PDEs for representing temporal data is expensive, TPPs were a good alternative. The drawback is that constructing an appropriate TPP for modeling temporal data requires the assumption of a particular temporal behavior of the data. To overcome this problem, Neural TPPs have been developed that enable learning of the parameters of the TPP. However, the research is relatively young for modeling dynamic graphs, and only a few TPPs have been proposed to handle edge-dynamic graphs. To allow for learning on a fully dynamic graph, we propose the first Marked Neural Spatio-Temporal Point Process (MNSTPP) that leverages a Dynamic Graph Neural Network to learn Spatio-TPPs to model and predict any event in a graph stream.
addition, our model can be updated efficiently by considering single events for local retraining.} }

%0 = inproceedings %A = Moallemy-Oureh, Alice and Beddar-Wiesing, Silvia and Nather, Rüdiger and Thomas, Josephine %B = Workshop on Temporal Graph Learning (TGL), NeurIPS %D = 2023 %T = Marked Neural Spatio-Temporal Point Process Involving a Dynamic Graph Neural Network %U = https://openreview.net/forum?id=QJx3Cmddsy

Typically, renewable-power-generation forecasting using machine learning involves creating separate models for each photovoltaic or wind park, known as single-task learning models. However, transfer learning has gained popularity in recent years, as it allows for the transfer of knowledge from source parks to target parks. Nevertheless, determining the most similar source park(s) for transfer learning can be challenging, particularly when the target park has limited or no historical data samples. To address this issue, we propose a multi-task learning architecture that employs a Unified Autoencoder (UAE) to initially learn a common representation of input weather features among tasks and then utilizes a Task-Embedding layer in a Neural Network (TENN) to learn task-specific information. This proposed UAE-TENN architecture can be easily extended to new parks with or without historical data. We evaluate the performance of our proposed architecture and compare it to single-task learning models on six photovoltaic and wind farm datasets consisting of a total of 529 parks. Our results show that the UAE-TENN architecture significantly improves power-forecasting performance by 10 to 19% for photovoltaic parks and 5 to 15% for wind parks compared to baseline models. We also demonstrate that UAE-TENN improves forecast accuracy for a new park by 19% for photovoltaic parks, even in a zero-shot learning scenario where there is no historical data. Additionally, we propose variants of the Unified Autoencoder with convolutional and LSTM layers, compare their performance, and provide a comparison among architectures with different numbers of task-embedding dimensions. Finally, we demonstrate the utility of trained task embeddings for interpretation and visualization purposes.

@article{nivarthi2023multi, author = {Nivarthi, Chandana Priya and Vogt, Stephan and Sick, Bernhard}, title = {Multi-Task Representation Learning for Renewable-Power Forecasting: A Comparative Analysis of Unified Autoencoder Variants and Task-Embedding Dimensions}, journal = {Machine Learning and Knowledge Extraction (MAKE)}, publisher = {MDPI}, year = {2023}, volume = {5}, number = {3}, pages = {1214--1233}, url = {https://www.mdpi.com/2504-4990/5/3/62}, doi = {10.3390/make5030062}, keywords = {imported, itegpub, isac-www}, abstract = {Typically, renewable-power-generation forecasting using machine learning involves creating separate models for each photovoltaic or wind park, known as single-task learning models. However, transfer learning has gained popularity in recent years, as it allows for the transfer of knowledge from source parks to target parks. Nevertheless, determining the most similar source park(s) for transfer learning can be challenging, particularly when the target park has limited or no historical data samples. To address this issue, we propose a multi-task learning architecture that employs a Unified Autoencoder (UAE) to initially learn a common representation of input weather features among tasks and then utilizes a Task-Embedding layer in a Neural Network (TENN) to learn task-specific information. This proposed UAE-TENN architecture can be easily extended to new parks with or without historical data. We evaluate the performance of our proposed architecture and compare it to single-task learning models on six photovoltaic and wind farm datasets consisting of a total of 529 parks. Our results show that the UAE-TENN architecture significantly improves power-forecasting performance by 10 to 19% for photovoltaic parks and 5 to 15% for wind parks compared to baseline models. We also demonstrate that UAE-TENN improves forecast accuracy for a new park by 19% for photovoltaic parks, even in a zero-shot learning scenario where there is no historical data. Additionally, we propose variants of the Unified Autoencoder with convolutional and LSTM layers, compare their performance, and provide a comparison among architectures with different numbers of task-embedding dimensions. Finally, we demonstrate the utility of trained task embeddings for interpretation and visualization purposes.} }

%0 = article %A = Nivarthi, Chandana Priya and Vogt, Stephan and Sick, Bernhard %D = 2023 %I = MDPI %T = Multi-Task Representation Learning for Renewable-Power Forecasting: A Comparative Analysis of Unified Autoencoder Variants and Task-Embedding Dimensions %U = https://www.mdpi.com/2504-4990/5/3/62

Anomaly detection plays a pivotal role in diverse realworld applications such as cybersecurity, fault detection, network
nitoring, predictive maintenance, and highly automated driving. However, obtaining labeled anomalous data can be a formidable
allenge, especially when anomalies exhibit temporal evolution. This paper introduces LATAM (Long short-term memory Autoencoder with Temporal Attention Mechanism) for few-shot anomaly detection, with the aim of enhancing detection performance in scenarios with limited labeled anomaly data. LATAM effectively captures temporal dependencies and emphasizes significant patterns in multivariate time series data. In our investigation, we
mprehensively evaluate LATAM against other anomaly detection models, particularly assessing its capability in few-shot learning
enarios where we have minimal examples from the normal class and none from the anomalous class in the training data. Our
perimental results, derived from real-world photovoltaic inverter data, highlight LATAM’s superiority, showcasing a substantial
% mean F1 score improvement, even when trained on a mere two-week dataset. Furthermore, LATAM demonstrates remarkable
sults on the open-source SWaT dataset, achieving a 12% boost in accuracy with only two days of training data. Moreover, we
troduce a simple yet effective dynamic thresholding mechanism, further enhancing the anomaly detection capabilities of LATAM.
is underscores LATAM’s efficacy in addressing the challenges posed by limited labeled anomalies in practical scenarios and it
oves valuable for downstream tasks involving temporal representation and time series prediction, extending its utility beyond
omaly detection applications.

@inproceedings{nivarthi2023towards, author = {Nivarthi, Chandana Priya and Sick, Bernhard}, title = {Towards Few-Shot Time Series Anomaly Detection with Temporal Attention and Dynamic Thresholding}, booktitle = {International Conference on Machine Learning and Applications (ICMLA)}, publisher = {IEEE}, year = {2023}, pages = {1444--1450}, doi = {10.1109/ICMLA58977.2023.00218}, keywords = {imported, itegpub, isac-www, few-shot, learning, anomaly, detection, temporal, attention, dynamic, thresholding}, abstract = {Anomaly detection plays a pivotal role in diverse realworld applications such as cybersecurity, fault detection, network
nitoring, predictive maintenance, and highly automated driving. However, obtaining labeled anomalous data can be a formidable
allenge, especially when anomalies exhibit temporal evolution. This paper introduces LATAM (Long short-term memory Autoencoder with Temporal Attention Mechanism) for few-shot anomaly detection, with the aim of enhancing detection performance in scenarios with limited labeled anomaly data. LATAM effectively captures temporal dependencies and emphasizes significant patterns in multivariate time series data. In our investigation, we
mprehensively evaluate LATAM against other anomaly detection models, particularly assessing its capability in few-shot learning
enarios where we have minimal examples from the normal class and none from the anomalous class in the training data. Our
perimental results, derived from real-world photovoltaic inverter data, highlight LATAM’s superiority, showcasing a substantial
% mean F1 score improvement, even when trained on a mere two-week dataset. Furthermore, LATAM demonstrates remarkable
sults on the open-source SWaT dataset, achieving a 12% boost in accuracy with only two days of training data. Moreover, we
troduce a simple yet effective dynamic thresholding mechanism, further enhancing the anomaly detection capabilities of LATAM.
is underscores LATAM’s efficacy in addressing the challenges posed by limited labeled anomalies in practical scenarios and it
oves valuable for downstream tasks involving temporal representation and time series prediction, extending its utility beyond
omaly detection applications.} }

%0 = inproceedings %A = Nivarthi, Chandana Priya and Sick, Bernhard %B = International Conference on Machine Learning and Applications (ICMLA) %D = 2023 %I = IEEE %T = Towards Few-Shot Time Series Anomaly Detection with Temporal Attention and Dynamic Thresholding

@book{tomforde2023organic,, title = {Organic Computing -- Doctoral Dissertation Colloquium 2022}, editor = {Tomforde, Sven and Krupitzer, Christian}, series = {Intelligent Embedded Systems}, publisher = {kassel university press}, year = {2023}, volume = {24}, doi = {10.17170/kobra-202302107484}, keywords = {imported, itegpub, isac-www} }

%0 = book %D = 2023 %I = kassel university press %T = Organic Computing -- Doctoral Dissertation Colloquium 2022

Due to the ongoing trend towards a decarbonisation of energy use, the power system is expected to become the backbone of all energy sectors and thus the fundamental critical infrastructure. High penetration with distributed energy resources demands the coordination of a large number of prosumers, partly controlled by home energy management systems (HEMS), to be designed in such a way that the power system’s operational limits are not violated. On the grid level, distribution management systems (DMS) seek to keep the power system in the normal operational state. On the prosumer level, distributed HEMS optimise the internal power flows by setpoint specification of batteries, photovoltaic generators, or flexible loads. The vision of the ODiS (Organic Distribution System) initiative is to develop an architecture to operate a distribution grid reliably, with high resiliency, and fully autonomously by developing “organic” HEMS and DMS which possess multiple self-x capabilities, collectively referred to as self-management. Thus, ODiS seeks answers to the following question: How can we create the most appropriate models, techniques, and algorithms to develop novel kinds of self-configuring, self-organising, self-healing, and self-optimising DMS that are integrally coupled with the distributed HEMS? In this concept paper, the vision of ODiS is presented in detail based on a thorough review of the state of the art.

@article{loeser2022vision, author = {Loeser, Inga and Braun, Martin and Gruhl, Christian and Menke, Jan-Hendrik and Sick, Bernhard and Tomforde, Sven}, title = {The Vision of Self-Management in Cognitive Organic Power Distribution Systems}, journal = {Energies}, publisher = {MDPI}, year = {2022}, volume = {15}, number = {3}, pages = {881}, url = {https://www.mdpi.com/1996-1073/15/3/881}, doi = {10.3390/en15030881}, keywords = {cognitive_energysystems, organic_computing, power_system, home_energy_management_system, distribution_management_system, self-configuration, self-organisation, self-healing, self-optimisation, imported, itegpub, isac-www}, abstract = {Due to the ongoing trend towards a decarbonisation of energy use, the power system is expected to become the backbone of all energy sectors and thus the fundamental critical infrastructure. High penetration with distributed energy resources demands the coordination of a large number of prosumers, partly controlled by home energy management systems (HEMS), to be designed in such a way that the power system’s operational limits are not violated. On the grid level, distribution management systems (DMS) seek to keep the power system in the normal operational state. On the prosumer level, distributed HEMS optimise the internal power flows by setpoint specification of batteries, photovoltaic generators, or flexible loads. The vision of the ODiS (Organic Distribution System) initiative is to develop an architecture to operate a distribution grid reliably, with high resiliency, and fully autonomously by developing “organic” HEMS and DMS which possess multiple self-x capabilities, collectively referred to as self-management. Thus, ODiS seeks answers to the following question: How can we create the most appropriate models, techniques, and algorithms to develop novel kinds of self-configuring, self-organising, self-healing, and self-optimising DMS that are integrally coupled with the distributed HEMS? In this concept paper, the vision of ODiS is presented in detail based on a thorough review of the state of the art.} }

%0 = article %A = Loeser, Inga and Braun, Martin and Gruhl, Christian and Menke, Jan-Hendrik and Sick, Bernhard and Tomforde, Sven %D = 2022 %I = MDPI %T = The Vision of Self-Management in Cognitive Organic Power Distribution Systems %U = https://www.mdpi.com/1996-1073/15/3/881

Stream-based active learning (AL) strategies minimize the labeling effort by querying labels that improve the classifier's performance the most. So far, these strategies neglect the fact that an oracle or expert requires time to provide a queried label. We show that existing AL methods deteriorate or even fail under the influence of such verification latency. The problem with these methods is that they estimate a label's utility on the currently available labeled data. However, when this label would arrive, some of the current data may have gotten outdated and new labels have arrived. In this article, we propose to simulate the available data at the time when the label would arrive. Therefore, our method Forgetting and Simulating (FS) forgets outdated information and simulates the delayed labels to get more realistic utility estimates. We assume to know the label's arrival date a priori and the classifier's training data to be bounded by a sliding window. Our extensive experiments show that FS improves stream-based AL strategies in settings with both, constant and variable verification latency.

@article{pham2022stream, author = {Pham, Tuan and Kottke, Daniel and Krempl, Georg and Sick, Bernhard}, title = {Stream-based active learning for sliding windows under the influence of verification latency}, journal = {Machine Learning}, publisher = {Springer}, year = {2022}, volume = {111}, number = {6}, pages = {2011--2036}, doi = {doi.org/10.1007/s10994-021-06099-z}, keywords = {imported, itegpub, isac-www}, abstract = {Stream-based active learning (AL) strategies minimize the labeling effort by querying labels that improve the classifier's performance the most. So far, these strategies neglect the fact that an oracle or expert requires time to provide a queried label. We show that existing AL methods deteriorate or even fail under the influence of such verification latency. The problem with these methods is that they estimate a label's utility on the currently available labeled data. However, when this label would arrive, some of the current data may have gotten outdated and new labels have arrived. In this article, we propose to simulate the available data at the time when the label would arrive. Therefore, our method Forgetting and Simulating (FS) forgets outdated information and simulates the delayed labels to get more realistic utility estimates. We assume to know the label's arrival date a priori and the classifier's training data to be bounded by a sliding window. Our extensive experiments show that FS improves stream-based AL strategies in settings with both, constant and variable verification latency. } }

%0 = article %A = Pham, Tuan and Kottke, Daniel and Krempl, Georg and Sick, Bernhard %D = 2022 %I = Springer %T = Stream-based active learning for sliding windows under the influence of verification latency

@book{tomforde2022organic,, title = {Organic Computing -- Doctoral Dissertation Colloquium 2021}, editor = {Tomforde, Sven and Krupitzer, Christian}, series = {Intelligent Embedded Systems}, publisher = {kassel university press}, year = {2022}, volume = {20}, doi = {10.17170/kobra-202202215780}, keywords = {imported, itegpub, isac-www} }

%0 = book %D = 2022 %I = kassel university press %T = Organic Computing -- Doctoral Dissertation Colloquium 2021