Note
Click here to download the full example code
Usage of the model-versus-data code verification tool on a Finite-Element (Abaqus) model¶
Check an Abaqus cantilever beam model versus a reference dataset with the 'CodeVerificationAgainstData' tool.
from __future__ import annotations
import logging
from gemseo.datasets.io_dataset import IODataset
from gemseo.utils.directory_creator import DirectoryNamingMethod
from vimseo import EXAMPLE_RUNS_DIR
from vimseo.api import activate_logger
from vimseo.api import create_model
from vimseo.core.model_settings import IntegratedModelSettings
from vimseo.tools.verification.verification_vs_data import CodeVerificationAgainstData
We first define the logger level:
activate_logger(level=logging.INFO)
Then we create the model to verify:
model_name = "BendingTestAnalytical"
load_case = "Cantilever"
model = create_model(
model_name,
load_case,
model_options=IntegratedModelSettings(
directory_archive_root=EXAMPLE_RUNS_DIR / "archive/verification_vs_data",
directory_scratch_root=EXAMPLE_RUNS_DIR / "scratch/verification_vs_data",
cache_file_path=EXAMPLE_RUNS_DIR
/ f"caches/verification_vs_data/{model_name}_{load_case}_cache.hdf",
),
)
Out:
INFO - 23:52:41: Found 2 entries in the cache file : C:\Users\sebastien.bocquet\Dev\vimseo\docs\runnable_examples\model_runs\caches\verification_vs_data\BendingTestAnalytical_Cantilever_cache.hdf node : node
We also need a reference dataset. Here we do it programmatically, but we can also create it from a csv file: ``
reference_data = IODataset().from_array(
data=[[10.0, 10.0, -4.0, -12.0], [15.0, 10.0, -6.0, -40.0]],
variable_names=["height", "width", "maximum_dplt", "reaction_forces"],
variable_names_to_group_names={
"height": "inputs",
"width": "inputs",
"maximum_dplt": "outputs",
"reaction_forces": "outputs",
},
)
All inputs to the verification are now available. We create the verification tool we are interested in.
verificator = CodeVerificationAgainstData(
directory_naming_method=DirectoryNamingMethod.NUMBERED,
working_directory="CodeVerificationAgainstData_results",
)
# The options can be modified.
# Alternatively, options can be passed as keyword arguments to
# ``CodeVerificationAgainstModelFromParameterSpace()`` constructor.
verificator.options["metric_names"] = [
"SquaredErrorMetric",
"RelativeErrorMetric",
"AbsoluteErrorMetric",
]
verificator.execute(
model=model,
reference_data=reference_data,
output_names=["maximum_dplt", "reaction_forces"],
description={
"title": "Verification of a cantilever analytic beam for a variation of beam height.",
"element_wise": ["Small height value", "High height value"],
},
)
Out:
INFO - 23:52:41: Working directory is C:\Users\sebastien.bocquet\Dev\vimseo\docs\runnable_examples\03_verification_vs_data\CodeVerificationAgainstData_results
INFO - 23:52:41: Working directory is C:\Users\sebastien.bocquet\Dev\vimseo\docs\runnable_examples\03_verification_vs_data\CodeVerificationAgainstData_results\CustomDOETool
INFO - 23:52:41: Working directory is C:\Users\sebastien.bocquet\Dev\vimseo\docs\runnable_examples\03_verification_vs_data\CodeVerificationAgainstData_results\CustomDOETool
INFO - 23:52:41:
INFO - 23:52:41: *** Start DOEScenario execution ***
INFO - 23:52:41: DOEScenario
INFO - 23:52:41: Disciplines: Model BendingTestAnalytical: An analytical model for the bending of a parallelepipedic beam
INFO - 23:52:41:
INFO - 23:52:41: Load case:
INFO - 23:52:41: Load case Cantilever: A cantilever load case.
INFO - 23:52:41:
INFO - 23:52:41: Boundary condition variables:
INFO - 23:52:41: ['imposed_dplt', 'relative_dplt_location']
INFO - 23:52:41:
INFO - 23:52:41: Plot parameters:
INFO - 23:52:41: {
INFO - 23:52:41: "curves": []
INFO - 23:52:41: }
INFO - 23:52:41: Load:
INFO - 23:52:41: Load(direction='', sign='', type='')
INFO - 23:52:41:
INFO - 23:52:41: Default values:
INFO - 23:52:41:
INFO - 23:52:41: Default geometrical variables:
INFO - 23:52:41: {"height": [40.0], "length": [600.0], "width": [30.0]}
INFO - 23:52:41:
INFO - 23:52:41: Default numerical variables:
INFO - 23:52:41: {}
INFO - 23:52:41:
INFO - 23:52:41: Default boundary conditions variables:
INFO - 23:52:41: {"imposed_dplt": [-5.0], "relative_dplt_location": [1.0]}
INFO - 23:52:41:
INFO - 23:52:41: Default material variables:
INFO - 23:52:41: {"nu_p": [0.3], "young_modulus": [210000.0]}
INFO - 23:52:41: model_inputs:
INFO - 23:52:41: [
INFO - 23:52:41: "length",
INFO - 23:52:41: "width",
INFO - 23:52:41: "height",
INFO - 23:52:41: "imposed_dplt",
INFO - 23:52:41: "relative_dplt_location",
INFO - 23:52:41: "young_modulus",
INFO - 23:52:41: "nu_p"
INFO - 23:52:41: ]
INFO - 23:52:41: model_outputs:
INFO - 23:52:41: [
INFO - 23:52:41: "reaction_forces",
INFO - 23:52:41: "moment",
INFO - 23:52:41: "moment_grid",
INFO - 23:52:41: "dplt",
INFO - 23:52:41: "dplt_grid",
INFO - 23:52:41: "maximum_dplt",
INFO - 23:52:41: "location_max_dplt",
INFO - 23:52:41: "dplt_at_force_location",
INFO - 23:52:41: "model",
INFO - 23:52:41: "load_case",
INFO - 23:52:41: "error_code",
INFO - 23:52:41: "description",
INFO - 23:52:41: "job_name",
INFO - 23:52:41: "persistent_result_files",
INFO - 23:52:41: "n_cpus",
INFO - 23:52:41: "date",
INFO - 23:52:41: "cpu_time",
INFO - 23:52:41: "user",
INFO - 23:52:41: "machine",
INFO - 23:52:41: "vims_git_version",
INFO - 23:52:41: "directory_archive_root",
INFO - 23:52:41: "directory_archive_job",
INFO - 23:52:41: "directory_scratch_root",
INFO - 23:52:41: "directory_scratch_job"
INFO - 23:52:41: ]
INFO - 23:52:41: MDO formulation: DisciplinaryOpt
INFO - 23:52:41: Optimization problem:
INFO - 23:52:41: minimize maximum_dplt(height, width)
INFO - 23:52:41: with respect to height, width
INFO - 23:52:41: over the design space:
INFO - 23:52:41: +--------+-------------+-------+-------------+-------+
INFO - 23:52:41: | Name | Lower bound | Value | Upper bound | Type |
INFO - 23:52:41: +--------+-------------+-------+-------------+-------+
INFO - 23:52:41: | height | -inf | None | inf | float |
INFO - 23:52:41: | width | -inf | None | inf | float |
INFO - 23:52:41: +--------+-------------+-------+-------------+-------+
INFO - 23:52:41: Solving optimization problem with algorithm CustomDOE:
INFO - 23:52:41: 50%|█████ | 1/2 [00:00<00:00, 12.60 it/sec, obj=-5]
INFO - 23:52:41: 100%|██████████| 2/2 [00:00<00:00, 17.30 it/sec, obj=-5]
INFO - 23:52:41: Optimization result:
INFO - 23:52:41: Optimizer info:
INFO - 23:52:41: Status: None
INFO - 23:52:41: Message: None
INFO - 23:52:41: Number of calls to the objective function by the optimizer: 2
INFO - 23:52:41: Solution:
INFO - 23:52:41: Objective: -4.999999999999999
INFO - 23:52:41: Design space:
INFO - 23:52:41: +--------+-------------+-------+-------------+-------+
INFO - 23:52:41: | Name | Lower bound | Value | Upper bound | Type |
INFO - 23:52:41: +--------+-------------+-------+-------------+-------+
INFO - 23:52:41: | height | -inf | 10 | inf | float |
INFO - 23:52:41: | width | -inf | 10 | inf | float |
INFO - 23:52:41: +--------+-------------+-------+-------------+-------+
INFO - 23:52:41: *** End DOEScenario execution (time: 0:00:00.123067) ***
VerificationResult(metadata=ToolResultMetadata(generic={'datetime': '03-09-2026_23-52-41', 'version': '0.1.7.dev12+gb7d116037'}, misc={}, settings={'input_names': [], 'output_names': ['maximum_dplt', 'reaction_forces'], 'metric_names': ['SquaredErrorMetric', 'RelativeErrorMetric', 'AbsoluteErrorMetric'], 'description': {'title': 'Verification of a cantilever analytic beam for a variation of beam height.', 'element_wise': ['Small height value', 'High height value']}}, report={}, model=ModelDescription(name='BendingTestAnalytical', summary=' An analytical model for the bending of a parallelepipedic beam', load_case=Beam_Cantilever(name='Cantilever', domain='Beam', summary='A cantilever load case.', plot_parameters=PlotParameters(curves=[]), bc_variable_names=['imposed_dplt', 'relative_dplt_location'], load=Load(direction='', sign='', type='')), dataflow={'model_inputs': ['length', 'width', 'height', 'imposed_dplt', 'relative_dplt_location', 'young_modulus', 'nu_p'], 'model_outputs': ['reaction_forces', 'moment', 'moment_grid', 'dplt', 'dplt_grid', 'maximum_dplt', 'location_max_dplt', 'dplt_at_force_location', 'model', 'load_case', 'error_code', 'description', 'job_name', 'persistent_result_files', 'n_cpus', 'date', 'cpu_time', 'user', 'machine', 'vims_git_version', 'directory_archive_root', 'directory_archive_job', 'directory_scratch_root', 'directory_scratch_job'], 'PreBendingTestAnalytical_Cantilever': {'inputs': ['length', 'width', 'height', 'imposed_dplt', 'relative_dplt_location', 'young_modulus', 'nu_p'], 'outputs': ['imposed_dplt_location', 'quadratic_moment', 'reaction_forces', 'moment', 'moment_grid', 'solver', 'boundary']}, 'RunBendingTestAnalytical': {'inputs': ['solver', 'boundary', 'moment', 'moment_grid', 'young_modulus', 'quadratic_moment', 'nu_p'], 'outputs': ['dplt', 'dplt_grid']}, 'PostBendingTestAnalytical_Cantilever': {'inputs': ['dplt', 'dplt_grid', 'moment', 'moment_grid', 'imposed_dplt_location', 'reaction_forces'], 'outputs': ['reaction_forces', 'maximum_dplt', 'dplt_grid', 'location_max_dplt', 'dplt', 'moment', 'moment_grid', 'dplt_at_force_location']}, 'subroutine_names': []}, default_inputs={<InputGroupNames.NUMERICAL_VARS: 'numerical variables'>: {}, <InputGroupNames.BC_VARS: 'boundary conditions variables'>: {'imposed_dplt': [-5.0], 'relative_dplt_location': [1.0]}, <InputGroupNames.GEOMETRICAL_VARS: 'geometrical variables'>: {'length': [600.0], 'width': [30.0], 'height': [40.0]}, <InputGroupNames.MATERIAL_VARS: 'material variables'>: {'young_modulus': [210000.0], 'nu_p': [0.3]}}, curves=[('dplt_grid', 'dplt'), ('moment_grid', 'moment')], verbose=False)), simulation_and_reference=GROUP inputs outputs Reference
VARIABLE height width cpu_time maximum_dplt reaction_forces maximum_dplt reaction_forces
COMPONENT 0 0 0 0 0 0 0
0 10.0 10.0 0.033884 -5.0 -12.152778 -4.0 -12.0
1 15.0 10.0 0.010976 -5.0 -41.015625 -6.0 -40.0, element_wise_metrics=GROUP SquaredErrorMetric RelativeErrorMetric AbsoluteErrorMetric inputs
VARIABLE maximum_dplt reaction_forces maximum_dplt reaction_forces maximum_dplt reaction_forces height width
COMPONENT 0 0 0 0 0 0 0 0
0 1.0 0.023341 0.250000 0.012731 1.0 0.152778 10.0 10.0
1 1.0 1.031494 0.166667 0.025391 1.0 1.015625 15.0 10.0, integrated_metrics={'SquaredErrorMetric': {'maximum_dplt': 1.0000000000000009, 'reaction_forces': 0.5274175950038581}, 'RelativeErrorMetric': {'maximum_dplt': 0.20833333328819448, 'reaction_forces': 0.019061053239892913}, 'AbsoluteErrorMetric': {'maximum_dplt': 1.0000000000000004, 'reaction_forces': 0.5842013888888893}}, description={'title': 'Verification of a cantilever analytic beam for a variation of beam height.', 'element_wise': ['Small height value', 'High height value']})
The result contains the error metrics:
verificator.result.integrated_metrics
Out:
{'SquaredErrorMetric': {'maximum_dplt': 1.0000000000000009, 'reaction_forces': 0.5274175950038581}, 'RelativeErrorMetric': {'maximum_dplt': 0.20833333328819448, 'reaction_forces': 0.019061053239892913}, 'AbsoluteErrorMetric': {'maximum_dplt': 1.0000000000000004, 'reaction_forces': 0.5842013888888893}}
And saved on disk, together with its metadata:
verificator.save_results()
Out:
INFO - 23:52:41: Saving result to C:\Users\sebastien.bocquet\Dev\vimseo\docs\runnable_examples\03_verification_vs_data\CodeVerificationAgainstData_results\CodeVerificationAgainstData_result.hdf5
INFO - 23:52:41: Saving result to C:\Users\sebastien.bocquet\Dev\vimseo\docs\runnable_examples\03_verification_vs_data\CodeVerificationAgainstData_results\CustomDOETool\CustomDOETool_result.hdf5
The saved results can be loaded in a dedicated dashboard to be explored.
The dashboard is opened by typing dashboard_verification in a terminal,
and selecting the tab Comparison case.
The results can also be plotted from the Python API. It shows the scatter matrix of the inputs:
figures = verificator.plot_results(
verificator.result,
"RelativeErrorMetric",
"reaction_forces",
save=False,
show=True,
directory_path=verificator.working_directory,
)
Out:
C:\Users\sebastien.bocquet\Dev\vimseo\.tox\doc\Lib\site-packages\pandas\plotting\_matplotlib\misc.py:100: UserWarning:
Attempting to set identical low and high xlims makes transformation singular; automatically expanding.
C:\Users\sebastien.bocquet\Dev\vimseo\.tox\doc\Lib\site-packages\pandas\plotting\_matplotlib\misc.py:101: UserWarning:
Attempting to set identical low and high ylims makes transformation singular; automatically expanding.
C:\Users\sebastien.bocquet\Dev\vimseo\.tox\doc\Lib\site-packages\pandas\plotting\_matplotlib\misc.py:91: UserWarning:
Attempting to set identical low and high xlims makes transformation singular; automatically expanding.
C:\Users\sebastien.bocquet\Dev\vimseo\.tox\doc\Lib\site-packages\gemseo\utils\matplotlib_figure.py:59: UserWarning:
FigureCanvasAgg is non-interactive, and thus cannot be shown
INFO - 23:52:42: Working directory is C:\Users\sebastien.bocquet\Dev\vimseo\docs\runnable_examples\03_verification_vs_data\CodeVerificationAgainstData_results
{'text/html': '<div> <script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG"></script><script type="text/javascript">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: "STIX-Web"}});}</script> <script type="text/javascript">window.PlotlyConfig = {MathJaxConfig: \'local\'};</script>\n <script charset="utf-8" src="https://cdn.plot.ly/plotly-2.35.0.min.js"></script> <div id="15161715-426b-429a-abb7-bb81c7bba2e9" class="plotly-graph-div" style="height:525px; width:100%;"></div> <script type="text/javascript"> window.PLOTLYENV=window.PLOTLYENV || {}; if (document.getElementById("15161715-426b-429a-abb7-bb81c7bba2e9")) { Plotly.newPlot( "15161715-426b-429a-abb7-bb81c7bba2e9", [{"alignmentgroup":"True","bingroup":"x","hovertemplate":"reaction_forces=%{x}\\u003cbr\\u003ecount=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e","legendgroup":"","marker":{"color":"#000001","pattern":{"shape":""}},"name":"","offsetgroup":"","orientation":"v","showlegend":false,"x":[0.01273148148042059,0.025390624999365237],"xaxis":"x","yaxis":"y","type":"histogram"}], {"template":{"data":{"candlestick":[{"decreasing":{"line":{"color":"#000033"}},"increasing":{"line":{"color":"#000032"}},"type":"candlestick"}],"contourcarpet":[{"colorscale":[[0.0,"#000011"],[0.1111111111111111,"#000012"],[0.2222222222222222,"#000013"],[0.3333333333333333,"#000014"],[0.4444444444444444,"#000015"],[0.5555555555555556,"#000016"],[0.6666666666666666,"#000017"],[0.7777777777777778,"#000018"],[0.8888888888888888,"#000019"],[1.0,"#000020"]],"type":"contourcarpet"}],"contour":[{"colorscale":[[0.0,"#000011"],[0.1111111111111111,"#000012"],[0.2222222222222222,"#000013"],[0.3333333333333333,"#000014"],[0.4444444444444444,"#000015"],[0.5555555555555556,"#000016"],[0.6666666666666666,"#000017"],[0.7777777777777778,"#000018"],[0.8888888888888888,"#000019"],[1.0,"#000020"]],"type":"contour"}],"heatmap":[{"colorscale":[[0.0,"#000011"],[0.1111111111111111,"#000012"],[0.2222222222222222,"#000013"],[0.3333333333333333,"#000014"],[0.4444444444444444,"#000015"],[0.5555555555555556,"#000016"],[0.6666666666666666,"#000017"],[0.7777777777777778,"#000018"],[0.8888888888888888,"#000019"],[1.0,"#000020"]],"type":"heatmap"}],"histogram2d":[{"colorscale":[[0.0,"#000011"],[0.1111111111111111,"#000012"],[0.2222222222222222,"#000013"],[0.3333333333333333,"#000014"],[0.4444444444444444,"#000015"],[0.5555555555555556,"#000016"],[0.6666666666666666,"#000017"],[0.7777777777777778,"#000018"],[0.8888888888888888,"#000019"],[1.0,"#000020"]],"type":"histogram2d"}],"icicle":[{"textfont":{"color":"white"},"type":"icicle"}],"sankey":[{"textfont":{"color":"#000036"},"type":"sankey"}],"scatter":[{"marker":{"line":{"width":0}},"type":"scatter"}],"table":[{"cells":{"fill":{"color":"#000038"},"font":{"color":"#000037"},"line":{"color":"#000039"}},"header":{"fill":{"color":"#000040"},"font":{"color":"#000036"},"line":{"color":"#000039"}},"type":"table"}],"waterfall":[{"connector":{"line":{"color":"#000036","width":2}},"decreasing":{"marker":{"color":"#000033"}},"increasing":{"marker":{"color":"#000032"}},"totals":{"marker":{"color":"#000034"}},"type":"waterfall"}]},"layout":{"coloraxis":{"colorscale":[[0.0,"#000011"],[0.1111111111111111,"#000012"],[0.2222222222222222,"#000013"],[0.3333333333333333,"#000014"],[0.4444444444444444,"#000015"],[0.5555555555555556,"#000016"],[0.6666666666666666,"#000017"],[0.7777777777777778,"#000018"],[0.8888888888888888,"#000019"],[1.0,"#000020"]]},"colorscale":{"diverging":[[0.0,"#000021"],[0.1,"#000022"],[0.2,"#000023"],[0.3,"#000024"],[0.4,"#000025"],[0.5,"#000026"],[0.6,"#000027"],[0.7,"#000028"],[0.8,"#000029"],[0.9,"#000030"],[1.0,"#000031"]],"sequential":[[0.0,"#000011"],[0.1111111111111111,"#000012"],[0.2222222222222222,"#000013"],[0.3333333333333333,"#000014"],[0.4444444444444444,"#000015"],[0.5555555555555556,"#000016"],[0.6666666666666666,"#000017"],[0.7777777777777778,"#000018"],[0.8888888888888888,"#000019"],[1.0,"#000020"]],"sequentialminus":[[0.0,"#000011"],[0.1111111111111111,"#000012"],[0.2222222222222222,"#000013"],[0.3333333333333333,"#000014"],[0.4444444444444444,"#000015"],[0.5555555555555556,"#000016"],[0.6666666666666666,"#000017"],[0.7777777777777778,"#000018"],[0.8888888888888888,"#000019"],[1.0,"#000020"]]},"colorway":["#000001","#000002","#000003","#000004","#000005","#000006","#000007","#000008","#000009","#000010"]}},"xaxis":{"anchor":"y","domain":[0.0,1.0],"title":{"text":"reaction_forces"}},"yaxis":{"anchor":"x","domain":[0.0,1.0],"title":{"text":"count"}},"legend":{"tracegroupgap":0},"margin":{"t":60},"barmode":"relative"}, {"responsive": true} ) }; </script> </div>'}
and an histogram of the errors:
figures["error_metric_histogram"]
Total running time of the script: ( 0 minutes 1.559 seconds)
Download Python source code: plot_bending_test_vs_data.py