1 year ago

#351601

test-img

Jay Antonio Oliver

Error reading netcdf and hdf files using python

I have the following code that would allow me to open .nc or .h5 files, and would supposidly open its contents.

import gdal
import glob
import os
import numpy as np
import pandas


def nch5read(fn, ncmaskscale=True):
    import netCDF4 as nc
    import h5py
    from pyhdf.SD import SD, SDC

    d = {'VARS':{}, 'ATTRS':{}}
    if os.path.splitext(fn)[1] == '.nc':
        f = nc.Dataset(fn)
        f.set_auto_maskandscale(ncmaskscale)
        for i in f.ncattrs():
            try:
                d['ATTRS'][i] = f.getncattr(i)
            except:
                pass

        for i in f.variables:
            try:
                d['VARS'][i] = {'DATA':np.squeeze(f.variables[i][:]),
                'ATTRS':{j:f.variables[i].getncattr(j) for j in f.variables
                }}
            except:
                pass

    if os.path.splitext(fn)[1]== '.h5':
        f = h5py.File(fn)
        d['ATTRS']=dict(f.attrs)

        for i in f:
            try:
                d['VARS'][i] = {'DATA':np.squeeze(f[i][()]),
                'ATTRS':dict(f[i].attrs)}
            except:
                pass

    return d

fn = "cons_dom.nc"

d = nch5read(fn)
print(d)

I am using an example domestic water use file, cons_dom.nc obtained from this website https://zenodo.org/record/897933#.YkH_SefMKUn but when running the code, I get a blank result {'VARS': {}, 'ATTRS': {}}

Additionally, I tried adding a section for .hdf into the code

import gdal
import glob
import os
import numpy as np
import pandas


def nch5read(fn, ncmaskscale=True):
    import netCDF4 as nc
    import h5py
    from pyhdf.SD import SD, SDC

    d = {'VARS':{}, 'ATTRS':{}}
    if os.path.splitext(fn)[1] == '.nc':
        f = nc.Dataset(fn)
        f.set_auto_maskandscale(ncmaskscale)
        for i in f.ncattrs():
            try:
                d['ATTRS'][i] = f.getncattr(i)
            except:
                pass

        for i in f.variables:
            try:
                d['VARS'][i] = {'DATA':np.squeeze(f.variables[i][:]),
                'ATTRS':{j:f.variables[i].getncattr(j) for j in f.variables
                }}
            except:
                pass

    if os.path.splitext(fn)[1]== '.h5':
        f = h5py.File(fn)
        d['ATTRS']=dict(f.attrs)

        for i in f:
            try:
                d['VARS'][i] = {'DATA':np.squeeze(f[i][()]),
                'ATTRS':dict(f[i].attrs)}
            except:
                pass
# Segment added to read .hdf files
    if os.path.splitext(fn)[1]== '.hdf':
        f = SD(fn, SDC.READ)
        f.set_auto_maskandscale(ncmaskscale)
        for i in f.variables:
            try:
                d['VARS'][i] = {'DATA':np.squeeze(f.variables[i][:]),
                'ATTRS':{j:f.variables[i].getncattr(j) for j in f.variables}}
            except:
                pass
    return d

fn = "MCD12Q1.A2016001.h10v05.006.2018149125004.hdf"

d = nch5read(fn)
print(d)

But when I try running a sample MODIS file, MCD12Q1.A2016001.h10v05.006.2018149125004.hdf I get an error

File "C:\Users\Jay_Oliver\Anaconda3\envs\rasterio\lib\site-packages\pyhdf\SD.py", line 3222, in _getattr
    index = a.index()
  File "C:\Users\Jay_Oliver\Anaconda3\envs\rasterio\lib\site-packages\pyhdf\SD.py", line 1193, in index
    _checkErr('find', self._index, 'illegal attribute name')
  File "C:\Users\Jay_Oliver\Anaconda3\envs\rasterio\lib\site-packages\pyhdf\error.py", line 23, in _checkErr
    raise HDF4Error(err)
pyhdf.error.HDF4Error: find (59): Invalid arguments to routine

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "nc_example.py", line 63, in <module>
    d = nch5read(fn)
  File "nc_example.py", line 52, in nch5read
    f.set_auto_maskandscale(ncmaskscale)
  File "C:\Users\Jay_Oliver\Anaconda3\envs\rasterio\lib\site-packages\pyhdf\SD.py", line 1446, in __getattr__
    return _getattr(self, name)
  File "C:\Users\Jay_Oliver\Anaconda3\envs\rasterio\lib\site-packages\pyhdf\SD.py", line 3224, in _getattr
    raise AttributeError("attribute not found")
AttributeError: attribute not found

python

netcdf

hdf

0 Answers

Your Answer

Accepted video resources