68 image lysozyme dataset recorded on the Jungfrau 16M detector at SwissFEL and formatted as a NeXus file, revised for clean cnxvalidate error report

This kit includes an additional revised master file, lyso009a_0087.JF07T32V01_master_rev.h5 that provides compliance with the October 2019 NXmx specification as proposed in https://github.com/HDRMX/definitions.git

To create a new NeXus master file, assuming DIALS is installed in the folder $DIALS, use this command:

libtbx.python $DIALS/modules/cctbx_project/xfel/swissfel/jf16m_cxigeom2nexus.py unassembled_file=lyso009a_0087.JF07T32V01.h5 geom_file=16M_bernina_backview_optimized_adu_quads.geom wavelength=1.368479 detector_distance=97.830 mask_file=lyso009a_0087.JF07T32V01.mask.h5

Geometry file is in CrystFEL format but has been realigned to group the modules hierarchically into quadrants.

View the data using DIALS: dials.image_viewer lyso009a_0087.JF07T32V01_master.h5

Process the data using DIALS, treating the images as stills, assuming 64 cores available on the system:
dials.stills_process mp.nproc=64 lyso009a_0087.JF07T32V01_master.h5 dispersion.gain=10 known_symmetry.space_group=P43212 known_symmetry.unit_cell=77,77,37,90,90,90 refinement_protocol.d_min_start=2.5

Download DIALS at dials.github.io.

After the DIALS run, for full NXmx compliance you will need the jungfrau portions of the script that was used to generate lyso009a_0087.JF07T32V01_master_rev.h5

#!/bin/bash
cp Therm_6_2.nxs Therm_6_2_rev.nxs
cp Therm_6_2_master.h5 Therm_6_2_master_rev.h5
cp jungfrau/lyso009a_0087.JF07T32V01_master.h5 jungfrau/lyso009a_0087.JF07T32V01_master_rev.h5
export curdat=`date +%FT%T.%3`
export LD_LIBRARY_PATH=$HOME/lib
export HDF5_PLUGIN_PATH=$HOME/lib
export PATH=$HOME/bin:$PATH
h5copy -i Therm_6_2_rev.nxs -o Therm_6_2_master_rev.h5 -s /entry/instrument/name -d /entry/instrument/name -f ref
h5copy -i Therm_6_2_rev.nxs -o Therm_6_2_master_rev.h5 -s /entry/instrument/source -d /entry/source -f ref
h5copy -i Therm_6_2_rev.nxs -o Therm_6_2_rev.nxs -s /entry/instrument/source -d /entry/source -f ref
h5copy -i jungfrau/lyso009a_0087.JF07T32V01_master.h5 -o jungfrau/lyso009a_0087.JF07T32V01_master_rev.h5 -s /entry/sample/beam -d /entry/instrument/beam -f ref
export end_time=`h5dump -d "/entry/end_time" Therm_6_2_master.h5 | grep ":" | sed 's/^.........//'|sed 's/.\$//'`
echo "end_time: $end_time"
python << 'EOL'
import h5py as h5
import numpy as np
import os
end_time=os.environ['end_time']
curdat=os.environ['curdat']
fvds = h5.File('Therm_6_2_rev.nxs','r+')
fmaster = h5.File('Therm_6_2_master_rev.h5','r+')
jungfrau= h5.File('jungfrau/lyso009a_0087.JF07T32V01_master_rev.h5','r+')
fvds_keys=fvds.keys()
fmaster_keys=fmaster.keys()
jungfrau_keys=jungfrau.keys()
fvds_entry=fvds['entry']
fmaster_entry=fmaster['entry']
jungfrau_entry=jungfrau['entry']
fvds_entry_keys=fvds_entry.keys()
fmaster_entry_keys=fmaster_entry.keys()
jungfrau_entry_keys=jungfrau_entry.keys()
fvds_entry_instrument=fvds['entry']['instrument']
fmaster_entry_instrument=fmaster['entry']['instrument']
jungfrau_entry_instrument=jungfrau['entry']['instrument']
fvds_entry_instrument_keys=fvds_entry_instrument.keys()
fmaster_entry_instrument_keys=fmaster_entry_instrument.keys()
jungfrau_entry_instrument_keys=jungfrau_entry_instrument.keys()
fvds_entry_instrument_name=(fvds['entry']['instrument']['name'])
fmaster_entry_instrument_name=(fmaster['entry']['instrument']['name'])
jungfrau['entry']['instrument'].create_dataset("name", data=np.string_("Paul Scherrer Institute SwissFEL Aramis 1 (Alvra)"))
jungfrau_entry_instrument_name=(jungfrau['entry']['instrument']['name'])
fvds_entry_instrument_short_name=fvds_entry_instrument.attrs['short_name']
fmaster_entry_instrument_short_name=fmaster_entry_instrument.attrs['short_name']
jungfrau_entry_instrument_name.attrs.modify('short_name',np.string_("Alvra"))
jungfrau_entry_instrument_short_name=jungfrau_entry_instrument_name.attrs['short_name']
zero_offset=fmaster_entry_instrument['detector']['module']['fast_pixel_direction'].attrs['offset']
fmaster_det_z=fmaster_entry_instrument['transformations']['det_z']
fvds_det_z=fvds_entry_instrument['transformations']['det_z']
print('fvds_keys: ',fvds_keys)
print('fmaster_keys: ',fmaster_keys)
print('jungfrau_keys: ',jungfrau_keys)
print('fvds_entry_keys: ',fvds_entry_keys)
print('fmaster_entry_keys: ',fmaster_entry_keys)
print('jungfrau_entry_keys: ',jungfrau_entry_keys)
print('fvds_entry_instrument_keys: ',fvds_entry_instrument_keys)
print('fmaster_entry_instrument_keys: ',fmaster_entry_instrument_keys)
print('jungfrau_entry_instrument_keys: ',jungfrau_entry_instrument_keys)
print('fvds_entry_instrument_name: ',fvds_entry_instrument_name)
print('fmaster_entry_instrument_name: ',fmaster_entry_instrument_name)
print('jungfrau_entry_instrument_name: ',jungfrau_entry_instrument_name)
print('fvds_entry_instrument_short_name: ',fvds_entry_instrument_short_name)
print('fmaster_entry_instrument_short_name: ',fmaster_entry_instrument_short_name)
print('jungfrau_entry_instrument_short_name: ',jungfrau_entry_instrument_short_name)
print('fmaster_entry_instrument_detector_module_fast_pixel_direction_offset: ',zero_offset)
print('fmaster_entry_instrument_detector_detector_z_det_z: ',fmaster_det_z)
print('fmaster_entry_end_time: ',end_time)
fmaster.attrs.modify('file_time',np.string_(end_time))
fmaster.attrs.modify('file_name',np.string_('Therm_6_2_master_rev.h5'))
fmaster.attrs.modify('HDF5_Version',np.string_('hdf5-1.8.18'))
fvds.attrs.modify('file_time',np.string_(end_time))
fvds.attrs.modify('file_name',np.string_('Therm_6_2_master_rev.h5'))
fvds.attrs.modify('HDF5_Version',np.string_('hdf5-1.10.5'))
jungfrau.attrs.modify('file_time',np.string_(curdat))
jungfrau.attrs.modify('file_name',np.string_('lyso009a_0087.JF07T32V01_master.h5'))
jungfrau.attrs.modify('HDF5_Version',np.string_('hdf5-1.10.5'))
fvds_entry_instrument_name.attrs.modify('short_name',np.string_(fvds_entry_instrument.attrs['short_name']))
fmaster_entry_instrument_name.attrs.modify('short_name',np.string_(fmaster_entry_instrument.attrs['short_name']))
fmaster_entry_instrument['attenuator']['attenuator_transmission'].attrs.modify('units',np.string_(""))
fmaster_entry_instrument['detector']['count_time'].attrs.modify('units',np.string_("s"))
fvds_entry_instrument_name.attrs.modify('short_name',np.string_(fvds_entry_instrument.attrs['short_name']))
fvds_entry_instrument['attenuator']['attenuator_transmission'].attrs.modify('units',np.string_(""))
fvds_entry_instrument['detector']['count_time'].attrs.modify('units',np.string_("s"))
fmaster_det_z.attrs.modify('offset',zero_offset)
fvds_det_z.attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['phi'].attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['chi'].attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['sam_x'].attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['sam_y'].attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['sam_z'].attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['omega'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['phi'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['chi'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['sam_x'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['sam_y'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['sam_z'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['omega'].attrs.modify('offset',zero_offset)
print(fmaster['entry']['instrument']['name'].attrs['short_name'])
print(fmaster['entry']['instrument']['name'].attrs['short_name'].shape)
print(fmaster['entry']['instrument']['name'].attrs['short_name'].dtype)
print("/entry/instrument/ELE_D0/pixel_mask_applied :",jungfrau_entry_instrument['ELE_D0']['pixel_mask_applied'])
del jungfrau_entry_instrument['ELE_D0']['pixel_mask_applied']
jungfrau_entry_instrument['ELE_D0'].create_dataset("pixel_mask_applied",dtype='int8', data=1)
print("/entry/instrument/ELE_D0/pixel_mask_applied :",jungfrau_entry_instrument['ELE_D0']['pixel_mask_applied'])
jungfrau_entry_source=jungfrau_entry.create_group('source')
jungfrau_entry_source=jungfrau_entry['source']
jungfrau_entry_source.attrs.modify('NX_class',np.string_("NXsource"))
jungfrau_entry_source.create_dataset("name",data=np.string_("Paul Scherrer Institute SwissFEL"))
jungfrau_entry_source['name'].attrs.modify('short_name',np.string_("SwissFEL"))
#jungfrau_entry_instrument.create_group['beam']
#jungfrau_entry_instrument['beam']=jungfrau_entry['sample']['beam']
jungfrau_entry_instrument['beam'].create_dataset('total_flux',dtype='float64',data=1000000000000.)
jungfrau_entry_instrument['beam']['total_flux'].attrs.modify('units',np.string_('/pulse'))
del jungfrau_entry['sample']['beam']
del fvds_entry_instrument.attrs['short_name']
del fmaster_entry_instrument.attrs['short_name']
del fmaster_entry_instrument['source']
fvds.close()
fmaster.close()
jungfrau.close()
quit()
EOL
$HOME/bin/nxvalidate -a NXmx -l /home/yaya/hdrmx_rev_29Sep19/hdrmx/definitions Therm_6_2_master_rev.h5
$HOME/bin/nxvalidate -a NXmx -l /home/yaya/hdrmx_rev_29Sep19/hdrmx/definitions Therm_6_2_rev.nxs
$HOME/bin/nxvalidate -a NXmx -l /home/yaya/hdrmx_rev_29Sep19/hdrmx/definitions jungfrau/lyso009a_0087.JF07T32V01_master_rev.h5