Huertas-Tato, Javier; Martín, Alejandro; Camacho, David
SILT: Efficient transformer training for inter-lingual inference Journal Article
In: Expert Systems with Applications, vol. 200, pp. 116923, 2022, ISSN: 0957-4174.
@article{huertas-tato_silt_2022,
title = {SILT: Efficient transformer training for inter-lingual inference},
author = {Javier Huertas-Tato and Alejandro Martín and David Camacho},
url = {https://www.sciencedirect.com/science/article/pii/S0957417422003578},
doi = {10.1016/j.eswa.2022.116923},
issn = {0957-4174},
year = {2022},
date = {2022-08-01},
urldate = {2022-08-01},
journal = {Expert Systems with Applications},
volume = {200},
pages = {116923},
abstract = {The ability of transformers to perform precision tasks such as question answering, Natural Language Inference (NLI) or summarizing, has enabled them to be ranked as one of the best paradigms to address Natural Language Processing (NLP) tasks. NLI is one of the best scenarios to test these architectures, due to the knowledge required to understand complex sentences and established relationships between a hypothesis and a premise. Nevertheless, these models suffer from the incapacity to generalize to other domains or from difficulties to face multilingual and interlingual scenarios. The leading pathway in the literature to address these issues involve designing and training extremely large architectures, but this causes unpredictable behaviors and establishes barriers which impede broad access and fine tuning. In this paper, we propose a new architecture called Siamese Inter-Lingual Transformer (SILT). This architecture is able to efficiently align multilingual embeddings for Natural Language Inference, allowing for unmatched language pairs to be processed. SILT leverages siamese pre-trained multi-lingual transformers with frozen weights where the two input sentences attend to each other to later be combined through a matrix alignment method. The experimental results carried out in this paper evidence that SILT allows to reduce drastically the number of trainable parameters while allowing for inter-lingual NLI and achieving state-of-the-art performance on common benchmarks.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Stevenson, Emma; Rodriguez-Fernandez, Victor; Urrutxua, Hodei; Camacho, David
Deep learning for all-vs-all conjunction detection Inproceedings
In: 5th Workshop on Key Topics in Orbit Propagation Applied to Space Situational Awareness (KePASSA), Logroño, Spain, 2022.
@inproceedings{stevenson2022_kepassa,
title = {Deep learning for all-vs-all conjunction detection},
author = {Emma Stevenson and Victor Rodriguez-Fernandez and Hodei Urrutxua and David Camacho},
year = {2022},
date = {2022-06-01},
urldate = {2022-06-01},
booktitle = {5th Workshop on Key Topics in Orbit Propagation Applied to Space Situational Awareness (KePASSA)},
address = {Logroño, Spain},
abstract = {This paper explores the use of different deep learning techniques for detecting conjunction events in an efficient and accurate way for improved space situational awareness. Framing the problem as a machine learning classification task, we present the performance of different data representations and model architectures on a realistic all-vs-all dataset generated using the CNES BAS3E space surveillance simulation framework, and compare the approaches to operationally used classical filters in screening performance and computational efficiency. Finally, we also investigate a novel methodology for improving the performance and generalisation ability of the models using a pre-trained orbit model, ORBERT, based on self-supervised learning techniques.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Huertas-García, Álvaro; Martín, Alejandro; Huertas-Tato, Javier; Camacho, David
Exploring Dimensionality Reduction Techniques in Multilingual Transformers Miscellaneous
CoRR, 2022.
@misc{nokey,
title = {Exploring Dimensionality Reduction Techniques in Multilingual Transformers},
author = {Álvaro Huertas-García and Alejandro Martín and Javier Huertas-Tato and David Camacho},
url = {https://doi.org/10.48550/arxiv.2204.08415},
doi = {10.48550/ARXIV.2204.08415},
year = {2022},
date = {2022-04-18},
urldate = {2022-04-18},
abstract = {Both in scientific literature and in industry,, Semantic and context-aware Natural Language Processing-based solutions have been gaining importance in recent years. The possibilities and performance shown by these models when dealing with complex Language Understanding tasks is unquestionable, from conversational agents to the fight against disinformation in social networks. In addition, considerable attention is also being paid to developing multilingual models to tackle the language bottleneck. The growing need to provide more complex models implementing all these features has been accompanied by an increase in their size, without being conservative in the number of dimensions required. This paper aims to give a comprehensive account of the impact of a wide variety of dimensional reduction techniques on the performance of different state-of-the-art multilingual Siamese Transformers, including unsupervised dimensional reduction techniques such as linear and nonlinear feature extraction, feature selection, and manifold techniques. In order to evaluate the effects of these techniques, we considered the multilingual extended version of Semantic Textual Similarity Benchmark (mSTSb) and two different baseline approaches, one using the pre-trained version of several models and another using their fine-tuned STS version. The results evidence that it is possible to achieve an average reduction in the number of dimensions of 91.58%±2.59% and 54.65%±32.20%, respectively. This work has also considered the consequences of dimensionality reduction for visualization purposes. The results of this study will significantly contribute to the understanding of how different tuning approaches affect performance on semantic-aware tasks and how dimensional reduction techniques deal with the high-dimensional embeddings computed for the STS task and their potential for highly demanding NLP tasks },
howpublished = {CoRR},
keywords = {},
pubstate = {published},
tppubtype = {misc}
}
Stevenson, Emma; Rodriguez-Fernandez, Victor; Minisci, Edmondo; Camacho, David
A deep learning approach to solar radio flux forecasting Journal Article
In: Acta Astronautica, vol. 193, pp. 595-606, 2022, ISSN: 0094-5765.
@article{STEVENSON2022595,
title = {A deep learning approach to solar radio flux forecasting},
author = {Emma Stevenson and Victor Rodriguez-Fernandez and Edmondo Minisci and David Camacho},
url = {https://www.sciencedirect.com/science/article/pii/S009457652100415X},
doi = {https://doi.org/10.1016/j.actaastro.2021.08.004},
issn = {0094-5765},
year = {2022},
date = {2022-01-01},
journal = {Acta Astronautica},
volume = {193},
pages = {595-606},
abstract = {The effect of atmospheric drag on spacecraft dynamics is considered one of the predominant sources of uncertainty in Low Earth Orbit. These effects are characterised in part by the atmospheric density, a quantity highly correlated to space weather. Current atmosphere models typically account for this through proxy indices such as the F10.7, but with variations in solar radio flux forecasts leading to significant orbit differences over just a few days, prediction of these quantities is a limiting factor in the accurate estimation of future drag conditions, and consequently orbital prediction. In this work, a novel deep residual architecture for univariate time series forecasting, N-BEATS, is employed for the prediction of the F10.7 solar proxy on the days-ahead timescales relevant to space operations. This untailored, pure deep learning approach has recently achieved state-of-the-art performance in time series forecasting competitions, outperforming well-established statistical, as well as statistical hybrid models, across a range of domains. The approach was found to be effective in single point forecasting up to 27-days ahead, and was additionally extended to produce forecast uncertainty estimates using deep ensembles. These forecasts were then compared to a persistence baseline and two operationally available forecasts: one statistical (provided by BGS, ESA), and one multi-flux neural network (by CLS, CNES). It was found that the N-BEATS model systematically outperformed the baseline and statistical approaches, and achieved an improved or similar performance to the multi-flux neural network approach despite only learning from a single variable.},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Huertas-García, Álvaro; Huertas-Tato, Javier; Martín, Alejandro; Camacho, David
CIVIC-UPM at CheckThat! 2021: Integration of Transformers in Misinformation Detection and Topic Classification Inproceedings
In: Conference and Labs of the Evaluation Forum (CLEF) Working Notes, pp. 520–530, 2021.
@inproceedings{huertas-garcia_civic-upm_2021,
title = {CIVIC-UPM at CheckThat! 2021: Integration of Transformers in Misinformation Detection and Topic Classification},
author = {Álvaro Huertas-García and Javier Huertas-Tato and Alejandro Martín and David Camacho},
url = {http://ceur-ws.org/Vol-2936/paper-41.pdf},
year = {2021},
date = {2021-05-24},
urldate = {2021-05-24},
booktitle = {Conference and Labs of the Evaluation Forum (CLEF) Working Notes},
pages = {520--530},
abstract = {Online Social Networks (OSNs) growth enables and amplifies the quick spread of harmful, manipulative and false information that influence public opinion while sow conflict on social or political issues. Therefore, the development of tools to detect malicious actors and to identify low-credibility information and misinformation sources is a new crucial challenge in the ever-evolving field of Artificial Intelligence. The scope of this paper is to present a Natural Language Processing (NLP) approach that uses Doc2Vec and different state-of-the-art transformer-based models for the CLEF2021 Checkthat! lab Task 3. Through this approach, the results show that it is possible to achieve 41.43% macro-average F1-score in the misinformation detection (Task A) and 67.65% macro-average F1-score in the topic classification (Task B).},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Stevenson, Emma; Rodriguez-Fernandez, Victor; Minisci, Edmondo; Camacho, David
A deep learning approach to space weather proxy forecasting for orbital prediction Inproceedings
In: 71st International Astronautical Congress (IAC), The CyberSpace Edition, 2020.
@inproceedings{stevenson2020_iac,
title = {A deep learning approach to space weather proxy forecasting for orbital prediction},
author = {Emma Stevenson and Victor Rodriguez-Fernandez and Edmondo Minisci and David Camacho},
url = {http://oa.upm.es/64345/},
year = {2020},
date = {2020-10-01},
booktitle = {71st International Astronautical Congress (IAC)},
address = {The CyberSpace Edition},
abstract = {The effect of atmospheric drag on spacecraft dynamics is considered one of the predominant sources of uncertainty in Low Earth Orbit. These effects are characterised in part by the atmospheric density, a quantity highly correlated to space weather. Current atmosphere models typically account for this through proxy indices such as the F10.7, but with variations in solar radio flux forecasts leading to significant orbit differences over just a few days, prediction of these quantities is a limiting factor in the accurate estimation of future drag conditions, and consequently orbital prediction. This has fundamental implications both in the short term, in the day-to-day management of operational spacecraft, and in the mid-to-long term, in determining satellite orbital lifetime. In this work, a novel deep residual architecture for univariate time series forecasting, N-BEATS, is employed for the prediction of the F10.7 solar proxy on the days-ahead timescales relevant to space operations. This untailored, pure deep learning approach has recently achieved state-of-the-art performance in time series forecasting competitions, outperforming well-established statistical, as well as statistical hybrid models, across a range of domains. The approach was found to be effective in single point forecasting up to 27-days ahead, and was additionally extended to produce forecast uncertainty estimates using deep ensembles. These forecasts were then compared to a persistence baseline and two operationally available forecasts: one statistical (provided by BGS, ESA), and one multi-flux neural network (by CLS, CNES). It was found that the N-BEATS model systematically outperformed the baseline and statistical approaches, and achieved an improved or similar performance to the multi-flux neural network approach despite only learning from a single variable},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Hernández, Alfonso; Panizo-LLedot, Ángel; Camacho, David
An ensemble algorithm based on deep learning for tuberculosis classification Inproceedings
In: International conference on intelligent data engineering and automated learning, pp. 145–154, Springer 2019.
@inproceedings{hernandez2019ensemble,
title = {An ensemble algorithm based on deep learning for tuberculosis classification},
author = {Alfonso Hernández and Ángel Panizo-LLedot and David Camacho},
doi = {10.1007/978-3-030-33607-3_17},
year = {2019},
date = {2019-01-01},
urldate = {2019-01-01},
booktitle = {International conference on intelligent data engineering and automated learning},
pages = {145--154},
organization = {Springer},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Martín, Alejandro; Fuentes-Hurtado, Félix; Naranjo, Valery; Camacho, David
Evolving deep neural networks architectures for Android malware classification Inproceedings
In: Evolutionary Computation (CEC), 2017 IEEE Congress on, pp. 1659–1666, IEEE 2017.
@inproceedings{martin2017evolving,
title = {Evolving deep neural networks architectures for Android malware classification},
author = {Alejandro Martín and Félix Fuentes-Hurtado and Valery Naranjo and David Camacho},
year = {2017},
date = {2017-01-01},
urldate = {2017-01-01},
booktitle = {Evolutionary Computation (CEC), 2017 IEEE Congress on},
pages = {1659--1666},
organization = {IEEE},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Martín, Alejandro; Lara-Cabrera, Raúl; Fuentes-Hurtado, Félix; Naranjo, Valery; Camacho, David
EvoDeep: a new Evolutionary approach for automatic Deep Neural Networks parametrisation Journal Article
In: Journal of Parallel and Distributed Computing, 2017.
@article{martin2017evodeep,
title = {EvoDeep: a new Evolutionary approach for automatic Deep Neural Networks parametrisation},
author = {Alejandro Martín and Raúl Lara-Cabrera and Félix Fuentes-Hurtado and Valery Naranjo and David Camacho},
year = {2017},
date = {2017-01-01},
urldate = {2017-01-01},
journal = {Journal of Parallel and Distributed Computing},
keywords = {},
pubstate = {published},
tppubtype = {article}
}