adds rastering and annealing metadata incl. UoM's

the file is generated into the "output" folder w/ .h5 extension
the most has been done already (probably)

.gitignore

@@ -1,8 +1,10 @@
 # ignores logs of h5tojson, jsontoh5
 *.log
 
-# ignores output json of main.py
+# ignores any output of main.py
 output/*.json
+output/*.h5
+output/*.nxs
 
 # ---> Python
 # Byte-compiled / optimized / DLL files

requirements.txt

@@ -1,2 +1,3 @@
 requests
 asyncio
+h5py

src/classes.py

@@ -13,17 +13,17 @@ class Layer:
     '''
     def __init__(self, layer_data):
         try:
+            self.operator = layer_data["fullname"]
             self.extra = layer_data["metadata_decoded"]["extra_fields"]
             self.layer_number = self.extra["Layer Progressive Number"]["value"] # integer
             self.target_elabid = self.extra["Target"]["value"] # elabid
             self.laser_system_elabid = self.extra["Laser System"]["value"] # elabid
             self.chamber_elabid = self.extra["Chamber"]["value"] # elabid
             self.rheed_system_elabid = self.extra["RHEED System"]["value"] # elabid
-            self.start_time = layer_data.get("created_at")
-            self.operator = layer_data.get("fullname")
-            self.description = layer_data.get("body")
             self.deposition_time = self.extra["Duration"]["value"]
+            self.deposition_time_unit = self.extra["Duration"]["unit"]
             self.repetition_rate = self.extra["Repetition rate"]["value"]
+            self.repetition_rate_unit = self.extra["Repetition rate"]["unit"]
             try:
                 self.number_of_pulses = (float(self.deposition_time) * float(self.repetition_rate)).__floor__()
             except ValueError:
@@ -34,15 +34,22 @@ class Layer:
     Please edit your eLabFTW entry and retry.
                 """)
             self.temperature = self.extra["Heater temperature"]["value"] # Note: this field used to have a trailing space in its name
+            self.temperature_unit = self.extra["Heater temperature"]["unit"]
             self.process_pressure = self.extra["Process pressure"]["value"] # Note: this field used to have a trailing space in its name
+            self.process_pressure_unit = self.extra["Process pressure"]["unit"]
             self.heating_method = self.extra["Heating Method"]["value"]
             self.layer_thickness = self.extra["Thickness"]["value"]
+            self.layer_thickness_unit = self.extra["Thickness"]["unit"]
             self.buffer_gas = self.extra["Buffer gas"]["value"]
             self.heater_target_distance = self.extra["Heater-target distance"]["value"]
+            self.heater_target_distance_unit = self.extra["Heater-target distance"]["unit"]
             self.laser_fluence = self.extra["Laser Intensity"]["value"] # here fluence = intensity
+            self.laser_fluence_unit = "J/(s cm^2)"
             self.laser_spot_area = self.extra["Spot Area"]["value"]
+            self.laser_spot_area_unit = "mm^2"
             try:
-                self.laser_energy = (float(self.laser_fluence) * float(self.laser_spot_area)).__round__(3)
+                self.laser_energy = ( float(self.laser_fluence) * float(self.laser_spot_area) / 100 ).__round__(3)
+                self.laser_energy_unit = "J/s"
             except ValueError:
                 # Since laser_energy is NOT required, if it can't be calculated warn user but allow the software to continue execution:
                 print("""
@@ -61,17 +68,27 @@ class Layer:
             self.pre_annealing_pressure = self.extra["Process pressure Pre"]["value"]
             self.pre_annealing_temperature = self.extra["Heater temperature Pre"]["value"]
             self.pre_annealing_duration = self.extra["Duration Pre"]["value"]
+            self.pre_annealing_pressure_unit = self.extra["Process pressure Pre"]["unit"]
+            self.pre_annealing_temperature_unit = self.extra["Heater temperature Pre"]["unit"]
+            self.pre_annealing_duration_unit = self.extra["Duration Pre"]["unit"]
             # Post annealing section
             self.post_annealing_ambient_gas = self.extra["Buffer gas PA"]["value"]
             self.post_annealing_pressure = self.extra["Process pressure PA"]["value"]
             self.post_annealing_temperature = self.extra["Heater temperature PA"]["value"]
             self.post_annealing_duration = self.extra["Duration PA"]["value"]
+            self.post_annealing_pressure_unit = self.extra["Process pressure PA"]["unit"]
+            self.post_annealing_temperature_unit = self.extra["Heater temperature PA"]["unit"]
+            self.post_annealing_duration_unit = self.extra["Duration PA"]["unit"]
+
             # Rejected but suggested by the NeXus standard:
             #self.laser_rastering_coefficients = None
         except KeyError as k:
             # Some keys are not required and can be called through the .get() method - which is permissive and allows null values;
             # Other keys are required so if they can't be called (invalid or null) raise error and stop execution of the program:
             raise KeyError(f"The provided dictionary lacks a \"{k}\" key. Check the deposition layer entry on eLabFTW and make sure you used the correct Experiment template.")
+        # Optional
+        self.start_time = layer_data.get("created_at") or None
+        self.description = layer_data.get("body") or None
     def get_instruments(self, apikey):
         raw_lasersys_data = APIHandler(apikey).get_entry_from_elabid(self.laser_system_elabid, entryType="items")
         raw_chamber_data = APIHandler(apikey).get_entry_from_elabid(self.chamber_elabid, entryType="items")
@@ -122,7 +139,13 @@ class Material:
             self.name = material_data["title"] # required
             self.extra = material_data["metadata_decoded"]["extra_fields"]
             self.compound_elabid = self.extra["Compound"]["value"]
-            self.dimensions = self.extra["Size"]["value"]
+            self.dimensions = str(self.extra["Size"]["value"]) # strings have a .count() method
+            if self.dimensions.count("mm") == 2:
+                self.dimensions_unit = "mm x mm"
+            elif self.dimensions[-1] == '"':
+                self.dimensions_unit = "inches"
+            else:
+                self.dimensions_unit = None
         except KeyError as k:
             # Some keys are not required and can be called through the .get() method - which is permissive and allows null values;
             # Other keys are required so if they can't be called (invalid or null) raise error and stop execution of the program:
@@ -149,9 +172,11 @@ class Substrate(Material):
         try:
             self.orientation = self.extra["Orientation"]["value"]
             self.miscut_angle = self.extra["Miscut Angle"]["value"]
+            self.miscut_angle_unit = self.extra["Miscut Angle"]["unit"]
             self.miscut_direction = self.extra["Miscut Direction"]["value"]
             # Not present (yet) on eLabFTW for Substrates:
-            self.thickness = None #self.extra["Thickness"]["value"]
+            self.thickness = "" #self.extra["Thickness"]["value"]
+            self.thickness_unit = "μm" #self.extra["Thickness"]["unit"]
             self.surface_treatment = self.extra["Surface treatment"]["value"]
             self.manufacturer = self.extra["Supplier"]["value"]
             self.batch_id = self.extra["Batch ID"]["value"]
@@ -163,6 +188,7 @@ class Target(Material):
         super().__init__(material_data)
         try:
             self.thickness = self.extra["Thickness"]["value"]
+            self.thickness_unit = self.extra["Thickness"]["unit"]
             self.shape = self.extra["shape"]["value"]
             self.solid_form = self.extra["Solid form"]["value"]
             self.manufacturer = self.extra["Supplier"]["value"]

src/main.py

@@ -1,4 +1,4 @@
-import os, json, requests
+import os, json, requests, h5py
 from getpass import getpass
 from APIHandler import APIHandler
 from classes import *
@@ -135,9 +135,15 @@ def make_nexus_schema_dictionary(substrate_object, layers):
                 "name": substrate_object.name,
                 "chemical_formula" : substrate_object.get_compound_formula(apikey),
                 "orientation" : substrate_object.orientation,
-                "miscut_angle" : substrate_object.miscut_angle,
+                "miscut_angle" : {
+                    "value": substrate_object.miscut_angle,
+                    "units": substrate_object.miscut_angle_unit
+                },
                 "miscut_direction" : substrate_object.miscut_direction,
-                "thickness" : substrate_object.thickness,
+                "thickness" : {
+                    "value": substrate_object.thickness,
+                    "units": substrate_object.thickness_unit,
+                },
                 "dimensions" : substrate_object.dimensions,
                 "surface_treatment" : substrate_object.surface_treatment,
                 "manufacturer" : substrate_object.manufacturer,
@@ -157,11 +163,14 @@ def make_nexus_schema_dictionary(substrate_object, layers):
             "description" : target_object.description,
             "shape" : target_object.shape,
             "dimensions" : target_object.dimensions,
-            "thickness" : target_object.thickness,
+            "thickness" : {
+                "value": target_object.thickness,
+                "units": target_object.thickness_unit,
+            },
             "solid_form" : target_object.solid_form,
             "manufacturer" : target_object.manufacturer,
+            "batch_id" : target_object.name,
             # TO-DO: currently not available:
-            # "batch_id" : target_object.batch_id,
         }
         multilayer[name] = {
             "target": target_dict,
@@ -169,17 +178,44 @@ def make_nexus_schema_dictionary(substrate_object, layers):
             "operator": layer.operator,
             "description": layer.description,
             "number_of_pulses": layer.number_of_pulses,
-            "deposition_time": layer.deposition_time,
-            "temperature": layer.temperature,
+            "deposition_time": {
+                "value": layer.deposition_time,
+                "units": layer.deposition_time_unit,
+            },
+            "temperature": {
+                "value": layer.temperature,
+                "units": layer.temperature_unit,
+            },
             "heating_method": layer.heating_method,
-            "layer_thickness": layer.layer_thickness,
+            "layer_thickness": {
+                "value": layer.layer_thickness,
+                "units": layer.layer_thickness_unit,
+            },
             "buffer_gas": layer.buffer_gas,
-            "process_pressure": layer.process_pressure,
-            "heater_target_distance": layer.heater_target_distance,
-            "repetition_rate": layer.repetition_rate,
-            "laser_fluence": layer.laser_fluence,
-            "laser_spot_area": layer.laser_spot_area,
-            "laser_energy": layer.laser_energy,
+            "process_pressure": {
+                "value": layer.process_pressure,
+                "units": layer.process_pressure_unit,
+            },
+            "heater_target_distance": {
+                "value": layer.heater_target_distance,
+                "units": layer.heater_target_distance_unit,
+            },
+            "repetition_rate": {
+                "value": layer.repetition_rate,
+                "units": layer.repetition_rate_unit,
+            },
+            "laser_fluence": {
+                "value": layer.laser_fluence,
+                "units": layer.laser_fluence_unit,
+            },
+            "laser_spot_area": {
+                "value": layer.laser_spot_area,
+                "units": layer.laser_spot_area_unit,
+            },
+            "laser_energy": {
+                "value": layer.laser_energy,
+                "units": layer.laser_energy_unit,
+            },
             "laser_rastering": {
                 "geometry": layer.laser_rastering_geometry,
                 "positions": layer.laser_rastering_positions,
@@ -187,19 +223,166 @@ def make_nexus_schema_dictionary(substrate_object, layers):
             },
             "pre_annealing": {
                 "ambient_gas": layer.pre_annealing_ambient_gas,
-                "pressure": layer.pre_annealing_pressure,
-                "temperature": layer.pre_annealing_temperature,
-                "duration": layer.pre_annealing_duration,
+                "pressure": {
+                    "value": layer.pre_annealing_pressure,
+                    "units": layer.pre_annealing_pressure_unit,
+                },
+                "temperature": {
+                    "value": layer.pre_annealing_temperature,
+                    "units": layer.pre_annealing_temperature_unit,
+                },
+                "duration": {
+                    "value": layer.pre_annealing_duration,
+                    "units": layer.pre_annealing_duration_unit,
+                },
             },
             "post_annealing": {
                 "ambient_gas": layer.post_annealing_ambient_gas,
-                "pressure": layer.post_annealing_pressure,
-                "temperature": layer.post_annealing_temperature,
-                "duration": layer.post_annealing_duration,
+                "pressure": {
+                    "value": layer.post_annealing_pressure,
+                    "units": layer.post_annealing_pressure_unit,
+                },
+                "temperature": {
+                    "value": layer.post_annealing_temperature,
+                    "units": layer.post_annealing_temperature_unit,
+                },
+                "duration": {
+                    "value": layer.post_annealing_duration,
+                    "units": layer.post_annealing_duration_unit,
+                },
             },
         }
     return pld_fabrication
 
+def build_nexus_file(pld_fabrication, output_path):
+    # NOTE: look at the mail attachment from Emiliano...
+    with h5py.File(output_path, "w") as f:
+        nx_pld_entry = f.create_group("pld_fabrication")
+        nx_pld_entry.attrs["NX_class"] = "NXentry"
+
+        # Sample section
+        nx_sample = nx_pld_entry.create_group("sample")
+        nx_sample.attrs["NX_class"] = "NXsample"
+        sample_dict = pld_fabrication["sample"]
+
+        # Substrate sub-section
+        nx_substrate = nx_sample.create_group("substrate")
+        nx_substrate.attrs["NX_class"] = "NXsubentry"
+        substrate_dict = sample_dict["substrate"]
+        try:
+            # Substrate fields (datasets)
+            nx_substrate.create_dataset("name", data=substrate_dict["name"])
+            nx_substrate.create_dataset("chemical_formula", data=substrate_dict["chemical_formula"])
+            nx_substrate.create_dataset("orientation", data=substrate_dict["orientation"])
+            nx_substrate.create_dataset("miscut_angle", data=substrate_dict["miscut_angle"]["value"]) # float
+            nx_substrate["miscut_angle"].attrs["units"] = substrate_dict["miscut_angle"]["units"]
+            nx_substrate.create_dataset("miscut_direction", data=substrate_dict["miscut_direction"])
+            nx_substrate.create_dataset("thickness", data=substrate_dict["thickness"]["value"]) # float/int
+            nx_substrate["thickness"].attrs["units"] = substrate_dict["thickness"]["units"]
+            nx_substrate.create_dataset("dimensions", data=substrate_dict["dimensions"])
+            nx_substrate.create_dataset("surface_treatment", data=substrate_dict["surface_treatment"])
+            nx_substrate.create_dataset("manufacturer", data=substrate_dict["manufacturer"])
+            nx_substrate.create_dataset("batch_id", data=substrate_dict["batch_id"])
+        except TypeError as te:
+            # sooner or later I'll handle this too - not today tho
+            raise TypeError(te)
+
+        # Multilayer sub-section
+        nx_multilayer = nx_sample.create_group("multilayer")
+        nx_multilayer.attrs["NX_class"] = "NXsubentry"
+        multilayer_dict = sample_dict["multilayer"]
+        # Repeat FOR EACH LAYER:
+        for layer in multilayer_dict:
+            nx_layer = nx_multilayer.create_group(layer)
+            nx_layer.attrs["NX_class"] = "NXsubentry"
+            layer_dict = multilayer_dict[layer]
+            # Sub-groups of a layer
+            ## Target
+            nx_target = nx_layer.create_group("target")
+            nx_target.attrs["NX_class"] = "NXsample"
+            target_dict = layer_dict["target"]
+            ## Rastering and Annealing
+            nx_laser_rastering = nx_layer.create_group("laser_rastering")
+            nx_laser_rastering.attrs["NX_class"] = "NXprocess"
+            rastering_dict = layer_dict["laser_rastering"]
+            nx_pre_annealing = nx_layer.create_group("pre_annealing")
+            nx_pre_annealing.attrs["NX_class"] = "NXprocess"
+            pre_ann_dict = layer_dict["pre_annealing"]
+            nx_post_annealing = nx_layer.create_group("post_annealing")
+            nx_post_annealing.attrs["NX_class"] = "NXprocess"
+            post_ann_dict = layer_dict["post_annealing"]
+
+            ## Target metadata
+            try:
+                nx_target.create_dataset("name", data = target_dict["name"])
+                nx_target.create_dataset("chemical_formula", data = target_dict["chemical_formula"])
+                nx_target.create_dataset("description", data = target_dict["description"])
+                nx_target.create_dataset("shape", data = target_dict["shape"])
+                nx_target.create_dataset("dimensions", data = target_dict["dimensions"])
+                nx_target.create_dataset("thickness", data = target_dict["thickness"]["value"]) # float/int
+                nx_target["thickness"].attrs["units"] = target_dict["thickness"]["units"]
+                nx_target.create_dataset("solid_form", data = target_dict["solid_form"])
+                nx_target.create_dataset("manufacturer", data = target_dict["manufacturer"])
+                nx_target.create_dataset("batch_id", data = target_dict["batch_id"])
+            except TypeError as te:
+                raise TypeError(te)
+            ## Other layer-specific metadata
+            try:
+                nx_layer.create_dataset("start_time", data = layer_dict["start_time"])
+                nx_layer.create_dataset("operator", data = layer_dict["operator"])
+                nx_layer.create_dataset("number_of_pulses", data = layer_dict["number_of_pulses"])
+                nx_layer.create_dataset("deposition_time", data = layer_dict["deposition_time"]["value"])
+                nx_layer["deposition_time"].attrs["units"] = layer_dict["deposition_time"]["units"]
+                nx_layer.create_dataset("repetition_rate", data = layer_dict["repetition_rate"]["value"])
+                nx_layer["repetition_rate"].attrs["units"] = layer_dict["repetition_rate"]["units"]
+                nx_layer.create_dataset("temperature", data = layer_dict["temperature"]["value"])
+                nx_layer["temperature"].attrs["units"] = layer_dict["temperature"]["units"]
+                nx_layer.create_dataset("heating_method", data = layer_dict["heating_method"])
+                nx_layer.create_dataset("layer_thickness", data = layer_dict["layer_thickness"]["value"])
+                nx_layer["layer_thickness"].attrs["units"] = layer_dict["layer_thickness"]["units"]
+                nx_layer.create_dataset("buffer_gas", data = layer_dict["buffer_gas"])
+                nx_layer.create_dataset("process_pressure", data = layer_dict["process_pressure"]["value"])
+                nx_layer["process_pressure"].attrs["units"] = layer_dict["process_pressure"]["units"]
+                nx_layer.create_dataset("heater_target_distance", data = layer_dict["heater_target_distance"]["value"])
+                nx_layer["heater_target_distance"].attrs["units"] = layer_dict["heater_target_distance"]["units"]
+                nx_layer.create_dataset("laser_fluence", data = layer_dict["laser_fluence"]["value"])
+                nx_layer["laser_fluence"].attrs["units"] = layer_dict["laser_fluence"]["units"]
+                nx_layer.create_dataset("laser_spot_area", data = layer_dict["laser_spot_area"]["value"])
+                nx_layer["laser_spot_area"].attrs["units"] = layer_dict["laser_spot_area"]["units"]
+                nx_layer.create_dataset("laser_energy", data = layer_dict["laser_energy"]["value"])
+                nx_layer["laser_energy"].attrs["units"] = layer_dict["laser_energy"]["units"]
+            except TypeError as te:
+                raise TypeError(te)
+            ## Rastering metadata
+            try:
+                nx_laser_rastering.create_dataset("geometry", data = rastering_dict["geometry"])
+                nx_laser_rastering.create_dataset("positions", data = rastering_dict["positions"])
+                nx_laser_rastering.create_dataset("velocities", data = rastering_dict["velocities"])
+            except TypeError as te:
+                raise TypeError(te)
+            ## Annealing metadata
+            try:
+                nx_pre_annealing.create_dataset("ambient_gas", data = pre_ann_dict["ambient_gas"])
+                nx_pre_annealing.create_dataset("pressure", data = pre_ann_dict["pressure"]["value"])
+                nx_pre_annealing["pressure"].attrs["units"] = pre_ann_dict["pressure"]["units"]
+                nx_pre_annealing.create_dataset("temperature", data = pre_ann_dict["temperature"]["value"])
+                nx_pre_annealing["temperature"].attrs["units"] = pre_ann_dict["temperature"]["units"]
+                nx_pre_annealing.create_dataset("duration", data = pre_ann_dict["duration"]["value"])
+                nx_pre_annealing["duration"].attrs["units"] = pre_ann_dict["duration"]["units"]
+            except TypeError as te:
+                raise TypeError(te)
+            try:
+                nx_post_annealing.create_dataset("ambient_gas", data = post_ann_dict["ambient_gas"])
+                nx_post_annealing.create_dataset("pressure", data = post_ann_dict["pressure"]["value"])
+                nx_post_annealing["pressure"].attrs["units"] = post_ann_dict["pressure"]["units"]
+                nx_post_annealing.create_dataset("temperature", data = post_ann_dict["temperature"]["value"])
+                nx_post_annealing["temperature"].attrs["units"] = post_ann_dict["temperature"]["units"]
+                nx_post_annealing.create_dataset("duration", data = post_ann_dict["duration"]["value"])
+                nx_post_annealing["duration"].attrs["units"] = post_ann_dict["duration"]["units"]
+            except TypeError as te:
+                raise TypeError(te)
+    return
+
 if __name__=="__main__":
     # TO-DO: place the API base URL somewhere else. 
     ELABFTW_API_URL = "https://elabftw.fisica.unina.it/api/v2"
@@ -207,9 +390,11 @@ if __name__=="__main__":
     elabid = input("Enter elabid of your starting sample [default= 1111]: ") or 1111
     data = APIHandler(apikey).get_entry_from_elabid(elabid)
     sample = Entrypoint(data)
+    sample_name = sample.name.strip().replace(" ","_")
     substrate_object = chain_entrypoint_to_batch(sample) # Substrate-class object
     layers = chain_entrypoint_to_layers(sample) # list of Layer-class objects
     result = make_nexus_schema_dictionary(substrate_object, layers)
     # print(make_nexus_schema_dictionary(substrate_object, layers)) # debug
-    with open (f"output/sample-{elabid}.json", "w") as f:
+    with open (f"output/sample-{sample_name}.json", "w") as f:
         json.dump(result, f, indent=3)
+    build_nexus_file(result, output_path=f"output/sample-{sample_name}-nexus.h5")