more elegant solution for deduplicating instruments

also edits help for deduplicate_instruments... to better explain what it
does; also fixes small typo ('default=' instead of 'default ='), row 448

.gitignore

@@ -1,8 +1,10 @@
 # ignores logs of h5tojson, jsontoh5
 *.log
 
-# ignores output json of main.py
+# ignores any output of main.py
 output/*.json
+output/*.h5
+output/*.nxs
 
 # ---> Python
 # Byte-compiled / optimized / DLL files

src/classes.py

@@ -175,7 +175,8 @@ class Substrate(Material):
             self.miscut_angle_unit = self.extra["Miscut Angle"]["unit"]
             self.miscut_direction = self.extra["Miscut Direction"]["value"]
             # Not present (yet) on eLabFTW for Substrates:
-            self.thickness = None #self.extra["Thickness"]["value"]
+            self.thickness = "" #self.extra["Thickness"]["value"]
+            self.thickness_unit = "μm" #self.extra["Thickness"]["unit"]
             self.surface_treatment = self.extra["Surface treatment"]["value"]
             self.manufacturer = self.extra["Supplier"]["value"]
             self.batch_id = self.extra["Batch ID"]["value"]
@@ -194,7 +195,7 @@ class Target(Material):
         except KeyError as k:
             raise KeyError(f"The provided dictionary lacks a \"{k}\" key - which is specific for PLD targets. Check the {self.name} target entry on eLabFTW and make sure you used the correct Resource template.")
         # Non-required attributes:
-        self.description = material_data.get("body") or None
+        self.description = material_data.get("body") or ""
 
 
 

src/main.py

@@ -100,7 +100,9 @@ def chain_layer_to_target(layer_object):
 
 def deduplicate_instruments_from_layers(layers):
     '''
-    Takes a list of Layer-class objects and for each layer gets the instruments used (laser, depo chamber and RHEED), returns deduplicated list. Ideally, the lists should only contain one element.
+    Takes a list of Layer-class objects and for each layer gets the instruments used (laser, depo chamber and RHEED), returns dictionary with one item per category. This means that if more layers share the same instruments it returns a dictionary with just their names as strings (no lists or sub-dictionaries).
+    
+    If different layers have different instruments (e.g. laser systems) the user is prompted to only select one.
     '''
     lasers = []
     chambers = []
@@ -110,11 +112,57 @@ def deduplicate_instruments_from_layers(layers):
         lasers.append(instruments["laser_system"])
         chambers.append(instruments["deposition_chamber"])
         rheeds.append(instruments["rheed_system"])
+    ded_lasers = list( set( lasers ) )
+    ded_chambers = list( set( chambers ) )
+    ded_rheeds = list( set( rheeds ) )
+    elegant_dict = {
+        # Keep key names human readable since they're used in the messages of the following errors
+        "Laser Systems": ded_lasers,
+        "Deposition Chamber": ded_chambers,
+        "RHEED Systems": ded_rheeds
+    } # dictionary's name's a joke
+    updated_dict = {} # use this for containing the final dataset
+    for ded in elegant_dict:
+        if len(elegant_dict[ded]) == 0:
+            # if len of list is 0 - empty list - raise error
+            raise IndexError(f"Missing data: no Laser System, Chamber and/or RHEED System is specified in any of the Deposition-type experiments related to this sample. Fix this on eLabFTW before retrying. Affected list: {ded}.")
+        elif len(elegant_dict[ded]) > 1:
+            # if len of list is > 1 - too many values - allow the user to pick one
+            print("Warning: different instruments have been used for different layers - which is currently not allowed.")
+            # there's a better way to do this but I can't remember now for the life of me...
+            i = 0
+            while i < len(elegant_dict[ded]):
+                print(f"{i} - {elegant_dict[ded][i]}")
+                i += 1
+            ans = None
+            while not type(ans) == int or not ans in range(0, len(elegant_dict[ded])):
+                ans = input("Please pick one of the previous (0, 1, ...) [default = 0]: ") or "0"
+                if ans.isdigit():
+                    ans = int(ans)
+                continue # unnecessary?
+            updated_dict[ded] = elegant_dict[ded][ans]
+        elif elegant_dict[ded][0] in ["", 0, None]:
+            # if len is 1 BUT value is "", 0 or None raise error
+            raise ValueError(f"Missing data: a Laser System, Chamber and/or RHEED System which is specified across all the Deposition-type experiments related to this sample is either empty or invalid. Fix this on eLabFTW before retrying. Affected list: {ded}.")
+        else:
+            # if none of the previous (only 1 value), that single value is used
+            updated_dict[ded] = elegant_dict[ded][0]
     instruments_used_dict = {
-        "laser_system": list( set( lasers ) ),
-        "deposition_chamber": list( set( chambers ) ),
-        "rheed_system" : list( set( rheeds ) ),
+        "laser_system": updated_dict["Laser Systems"],
+        "deposition_chamber": updated_dict["Deposition Chamber"],
+        "rheed_system": updated_dict["RHEED Systems"],
     }
+    return instruments_used_dict
+    
+    ### OLD CODE
+    # if 0 in [ len(i) for i in elegant_list ]:
+    #     # i.e. if length of one of the lists in elegant_list is zero (missing data):
+    #     raise IndexError("Missing data: no Laser System, Chamber and/or RHEED System is specified in any of the Deposition-type experiments related to this sample.")
+    # if not all([ len(i) == 1 for i in elegant_list ]):
+    #     print("Warning: different instruments have been used for different layers - which is currently not allowed.")
+    #     # for every element in elegant list check if len > 1 and if it is 
+    #     print("Selecting the first occurence for every category...")
+    ###
     # lasers = { f"layer_{lyr.layer_number}": lyr.laser_system for lyr in layers }
     # chambers = { f"layer_{lyr.layer_number}": lyr.deposition_chamber for lyr in layers }
     # rheeds = { f"layer_{lyr.layer_number}": lyr.rheed_system for lyr in layers }
@@ -123,7 +171,6 @@ def deduplicate_instruments_from_layers(layers):
     #     "deposition_chamber": chambers,
     #     "rheed_system": rheeds,
     # }
-    return instruments_used_dict
 
 def make_nexus_schema_dictionary(substrate_object, layers):
     '''
@@ -135,9 +182,15 @@ def make_nexus_schema_dictionary(substrate_object, layers):
                 "name": substrate_object.name,
                 "chemical_formula" : substrate_object.get_compound_formula(apikey),
                 "orientation" : substrate_object.orientation,
-                "miscut_angle" : substrate_object.miscut_angle,
+                "miscut_angle" : {
+                    "value": substrate_object.miscut_angle,
+                    "units": substrate_object.miscut_angle_unit
+                },
                 "miscut_direction" : substrate_object.miscut_direction,
-                "thickness" : substrate_object.thickness,
+                "thickness" : {
+                    "value": substrate_object.thickness,
+                    "units": substrate_object.thickness_unit,
+                },
                 "dimensions" : substrate_object.dimensions,
                 "surface_treatment" : substrate_object.surface_treatment,
                 "manufacturer" : substrate_object.manufacturer,
@@ -157,11 +210,14 @@ def make_nexus_schema_dictionary(substrate_object, layers):
             "description" : target_object.description,
             "shape" : target_object.shape,
             "dimensions" : target_object.dimensions,
-            "thickness" : target_object.thickness,
+            "thickness" : {
+                "value": target_object.thickness,
+                "units": target_object.thickness_unit,
+            },
             "solid_form" : target_object.solid_form,
             "manufacturer" : target_object.manufacturer,
+            "batch_id" : target_object.name,
             # TO-DO: currently not available:
-            # "batch_id" : target_object.batch_id,
         }
         multilayer[name] = {
             "target": target_dict,
@@ -169,17 +225,44 @@ def make_nexus_schema_dictionary(substrate_object, layers):
             "operator": layer.operator,
             "description": layer.description,
             "number_of_pulses": layer.number_of_pulses,
-            "deposition_time": layer.deposition_time,
-            "temperature": layer.temperature,
+            "deposition_time": {
+                "value": layer.deposition_time,
+                "units": layer.deposition_time_unit,
+            },
+            "temperature": {
+                "value": layer.temperature,
+                "units": layer.temperature_unit,
+            },
             "heating_method": layer.heating_method,
-            "layer_thickness": layer.layer_thickness,
+            "layer_thickness": {
+                "value": layer.layer_thickness,
+                "units": layer.layer_thickness_unit,
+            },
             "buffer_gas": layer.buffer_gas,
-            "process_pressure": layer.process_pressure,
-            "heater_target_distance": layer.heater_target_distance,
-            "repetition_rate": layer.repetition_rate,
-            "laser_fluence": layer.laser_fluence,
-            "laser_spot_area": layer.laser_spot_area,
-            "laser_energy": layer.laser_energy,
+            "process_pressure": {
+                "value": layer.process_pressure,
+                "units": layer.process_pressure_unit,
+            },
+            "heater_target_distance": {
+                "value": layer.heater_target_distance,
+                "units": layer.heater_target_distance_unit,
+            },
+            "repetition_rate": {
+                "value": layer.repetition_rate,
+                "units": layer.repetition_rate_unit,
+            },
+            "laser_fluence": {
+                "value": layer.laser_fluence,
+                "units": layer.laser_fluence_unit,
+            },
+            "laser_spot_area": {
+                "value": layer.laser_spot_area,
+                "units": layer.laser_spot_area_unit,
+            },
+            "laser_energy": {
+                "value": layer.laser_energy,
+                "units": layer.laser_energy_unit,
+            },
             "laser_rastering": {
                 "geometry": layer.laser_rastering_geometry,
                 "positions": layer.laser_rastering_positions,
@@ -187,15 +270,33 @@ def make_nexus_schema_dictionary(substrate_object, layers):
             },
             "pre_annealing": {
                 "ambient_gas": layer.pre_annealing_ambient_gas,
-                "pressure": layer.pre_annealing_pressure,
-                "temperature": layer.pre_annealing_temperature,
-                "duration": layer.pre_annealing_duration,
+                "pressure": {
+                    "value": layer.pre_annealing_pressure,
+                    "units": layer.pre_annealing_pressure_unit,
+                },
+                "temperature": {
+                    "value": layer.pre_annealing_temperature,
+                    "units": layer.pre_annealing_temperature_unit,
+                },
+                "duration": {
+                    "value": layer.pre_annealing_duration,
+                    "units": layer.pre_annealing_duration_unit,
+                },
             },
             "post_annealing": {
                 "ambient_gas": layer.post_annealing_ambient_gas,
-                "pressure": layer.post_annealing_pressure,
-                "temperature": layer.post_annealing_temperature,
-                "duration": layer.post_annealing_duration,
+                "pressure": {
+                    "value": layer.post_annealing_pressure,
+                    "units": layer.post_annealing_pressure_unit,
+                },
+                "temperature": {
+                    "value": layer.post_annealing_temperature,
+                    "units": layer.post_annealing_temperature_unit,
+                },
+                "duration": {
+                    "value": layer.post_annealing_duration,
+                    "units": layer.post_annealing_duration_unit,
+                },
             },
         }
     return pld_fabrication
@@ -209,22 +310,149 @@ def build_nexus_file(pld_fabrication, output_path):
         # Sample section
         nx_sample = nx_pld_entry.create_group("sample")
         nx_sample.attrs["NX_class"] = "NXsample"
+        sample_dict = pld_fabrication["sample"]
 
-        # Substrate section
-        nx_substrate = nx_pld_entry.create_group("substrate")
+        # Substrate sub-section
+        nx_substrate = nx_sample.create_group("substrate")
         nx_substrate.attrs["NX_class"] = "NXsubentry"
-    pass
+        substrate_dict = sample_dict["substrate"]
+        try:
+            # Substrate fields (datasets)
+            nx_substrate.create_dataset("name", data=substrate_dict["name"])
+            nx_substrate.create_dataset("chemical_formula", data=substrate_dict["chemical_formula"])
+            nx_substrate.create_dataset("orientation", data=substrate_dict["orientation"])
+            nx_substrate.create_dataset("miscut_angle", data=substrate_dict["miscut_angle"]["value"]) # float
+            nx_substrate["miscut_angle"].attrs["units"] = substrate_dict["miscut_angle"]["units"]
+            nx_substrate.create_dataset("miscut_direction", data=substrate_dict["miscut_direction"])
+            nx_substrate.create_dataset("thickness", data=substrate_dict["thickness"]["value"]) # float/int
+            nx_substrate["thickness"].attrs["units"] = substrate_dict["thickness"]["units"]
+            nx_substrate.create_dataset("dimensions", data=substrate_dict["dimensions"])
+            nx_substrate.create_dataset("surface_treatment", data=substrate_dict["surface_treatment"])
+            nx_substrate.create_dataset("manufacturer", data=substrate_dict["manufacturer"])
+            nx_substrate.create_dataset("batch_id", data=substrate_dict["batch_id"])
+        except TypeError as te:
+            # sooner or later I'll handle this too - not today tho
+            raise TypeError(te)
+
+        # Multilayer sub-section
+        nx_multilayer = nx_sample.create_group("multilayer")
+        nx_multilayer.attrs["NX_class"] = "NXsubentry"
+        multilayer_dict = sample_dict["multilayer"]
+        # Repeat FOR EACH LAYER:
+        for layer in multilayer_dict:
+            nx_layer = nx_multilayer.create_group(layer)
+            nx_layer.attrs["NX_class"] = "NXsubentry"
+            layer_dict = multilayer_dict[layer]
+            # Sub-groups of a layer
+            ## Target
+            nx_target = nx_layer.create_group("target")
+            nx_target.attrs["NX_class"] = "NXsample"
+            target_dict = layer_dict["target"]
+            ## Rastering and Annealing
+            nx_laser_rastering = nx_layer.create_group("laser_rastering")
+            nx_laser_rastering.attrs["NX_class"] = "NXprocess"
+            rastering_dict = layer_dict["laser_rastering"]
+            nx_pre_annealing = nx_layer.create_group("pre_annealing")
+            nx_pre_annealing.attrs["NX_class"] = "NXprocess"
+            pre_ann_dict = layer_dict["pre_annealing"]
+            nx_post_annealing = nx_layer.create_group("post_annealing")
+            nx_post_annealing.attrs["NX_class"] = "NXprocess"
+            post_ann_dict = layer_dict["post_annealing"]
+
+            ## Target metadata
+            try:
+                nx_target.create_dataset("name", data = target_dict["name"])
+                nx_target.create_dataset("chemical_formula", data = target_dict["chemical_formula"])
+                nx_target.create_dataset("description", data = target_dict["description"])
+                nx_target.create_dataset("shape", data = target_dict["shape"])
+                nx_target.create_dataset("dimensions", data = target_dict["dimensions"])
+                nx_target.create_dataset("thickness", data = target_dict["thickness"]["value"]) # float/int
+                nx_target["thickness"].attrs["units"] = target_dict["thickness"]["units"]
+                nx_target.create_dataset("solid_form", data = target_dict["solid_form"])
+                nx_target.create_dataset("manufacturer", data = target_dict["manufacturer"])
+                nx_target.create_dataset("batch_id", data = target_dict["batch_id"])
+            except TypeError as te:
+                raise TypeError(te)
+            ## Other layer-specific metadata
+            try:
+                nx_layer.create_dataset("start_time", data = layer_dict["start_time"])
+                nx_layer.create_dataset("operator", data = layer_dict["operator"])
+                nx_layer.create_dataset("number_of_pulses", data = layer_dict["number_of_pulses"])
+                nx_layer.create_dataset("deposition_time", data = layer_dict["deposition_time"]["value"])
+                nx_layer["deposition_time"].attrs["units"] = layer_dict["deposition_time"]["units"]
+                nx_layer.create_dataset("repetition_rate", data = layer_dict["repetition_rate"]["value"])
+                nx_layer["repetition_rate"].attrs["units"] = layer_dict["repetition_rate"]["units"]
+                nx_layer.create_dataset("temperature", data = layer_dict["temperature"]["value"])
+                nx_layer["temperature"].attrs["units"] = layer_dict["temperature"]["units"]
+                nx_layer.create_dataset("heating_method", data = layer_dict["heating_method"])
+                nx_layer.create_dataset("layer_thickness", data = layer_dict["layer_thickness"]["value"])
+                nx_layer["layer_thickness"].attrs["units"] = layer_dict["layer_thickness"]["units"]
+                nx_layer.create_dataset("buffer_gas", data = layer_dict["buffer_gas"])
+                nx_layer.create_dataset("process_pressure", data = layer_dict["process_pressure"]["value"])
+                nx_layer["process_pressure"].attrs["units"] = layer_dict["process_pressure"]["units"]
+                nx_layer.create_dataset("heater_target_distance", data = layer_dict["heater_target_distance"]["value"])
+                nx_layer["heater_target_distance"].attrs["units"] = layer_dict["heater_target_distance"]["units"]
+                nx_layer.create_dataset("laser_fluence", data = layer_dict["laser_fluence"]["value"])
+                nx_layer["laser_fluence"].attrs["units"] = layer_dict["laser_fluence"]["units"]
+                nx_layer.create_dataset("laser_spot_area", data = layer_dict["laser_spot_area"]["value"])
+                nx_layer["laser_spot_area"].attrs["units"] = layer_dict["laser_spot_area"]["units"]
+                nx_layer.create_dataset("laser_energy", data = layer_dict["laser_energy"]["value"])
+                nx_layer["laser_energy"].attrs["units"] = layer_dict["laser_energy"]["units"]
+            except TypeError as te:
+                raise TypeError(te)
+            ## Rastering metadata
+            try:
+                nx_laser_rastering.create_dataset("geometry", data = rastering_dict["geometry"])
+                nx_laser_rastering.create_dataset("positions", data = rastering_dict["positions"])
+                nx_laser_rastering.create_dataset("velocities", data = rastering_dict["velocities"])
+            except TypeError as te:
+                raise TypeError(te)
+            ## Annealing metadata
+            try:
+                nx_pre_annealing.create_dataset("ambient_gas", data = pre_ann_dict["ambient_gas"])
+                nx_pre_annealing.create_dataset("pressure", data = pre_ann_dict["pressure"]["value"])
+                nx_pre_annealing["pressure"].attrs["units"] = pre_ann_dict["pressure"]["units"]
+                nx_pre_annealing.create_dataset("temperature", data = pre_ann_dict["temperature"]["value"])
+                nx_pre_annealing["temperature"].attrs["units"] = pre_ann_dict["temperature"]["units"]
+                nx_pre_annealing.create_dataset("duration", data = pre_ann_dict["duration"]["value"])
+                nx_pre_annealing["duration"].attrs["units"] = pre_ann_dict["duration"]["units"]
+            except TypeError as te:
+                raise TypeError(te)
+            try:
+                nx_post_annealing.create_dataset("ambient_gas", data = post_ann_dict["ambient_gas"])
+                nx_post_annealing.create_dataset("pressure", data = post_ann_dict["pressure"]["value"])
+                nx_post_annealing["pressure"].attrs["units"] = post_ann_dict["pressure"]["units"]
+                nx_post_annealing.create_dataset("temperature", data = post_ann_dict["temperature"]["value"])
+                nx_post_annealing["temperature"].attrs["units"] = post_ann_dict["temperature"]["units"]
+                nx_post_annealing.create_dataset("duration", data = post_ann_dict["duration"]["value"])
+                nx_post_annealing["duration"].attrs["units"] = post_ann_dict["duration"]["units"]
+            except TypeError as te:
+                raise TypeError(te)
+        
+        # Instruments used section
+        nx_instruments = nx_pld_entry.create_group("instruments_used")
+        nx_instruments.attrs["NX_class"] = "NXinstrument"
+        instruments_dict = pld_fabrication["instruments_used"]
+        try:
+            nx_instruments.create_dataset("laser_system", data = instruments_dict["laser_system"])
+            nx_instruments.create_dataset("deposition_chamber", data = instruments_dict["deposition_chamber"])
+            nx_instruments.create_dataset("rheed_system", data = instruments_dict["rheed_system"])
+        except TypeError as te:
+            raise TypeError(te)
+    return
 
 if __name__=="__main__":
     # TO-DO: place the API base URL somewhere else. 
     ELABFTW_API_URL = "https://elabftw.fisica.unina.it/api/v2"
     apikey = getpass("Paste API key here: ")
-    elabid = input("Enter elabid of your starting sample [default= 1111]: ") or 1111
+    elabid = input("Enter elabid of your starting sample [default = 1111]: ") or 1111
     data = APIHandler(apikey).get_entry_from_elabid(elabid)
     sample = Entrypoint(data)
+    sample_name = sample.name.strip().replace(" ","_")
     substrate_object = chain_entrypoint_to_batch(sample) # Substrate-class object
     layers = chain_entrypoint_to_layers(sample) # list of Layer-class objects
     result = make_nexus_schema_dictionary(substrate_object, layers)
     # print(make_nexus_schema_dictionary(substrate_object, layers)) # debug
-    with open (f"output/sample-{elabid}.json", "w") as f:
+    with open (f"output/sample-{sample_name}.json", "w") as f:
         json.dump(result, f, indent=3)
+    build_nexus_file(result, output_path=f"output/sample-{sample_name}-nexus.h5")