1 year ago

#360546

test-img

RomuloPBenedetti

GCC Cython refuses to link gfortran in outside conda environment

I'm trying to make a package that binds Fortran code through Cython work outside of conda environments, like Fedora 35 or a Ubuntu 18.04, the problem is that while building the extension, gcc refuses to link gfortran.

setup.py:

cmd = "make release"
subprocess.call(cmd, shell=True, stdout=open(os.devnull, 'wb'))

ext1 = Extension(
    name='model',
    sources=['src/cython/model.pyx'],
    extra_compile_args=['-g', "-static"],
    libraries=['gfortran'],
    extra_link_args=[
        "build/release/Utils.o", 
        "build/release/Logger.o", 
        ...
    ]
)

setup(
  ...
  ext_modules=cythonize([ext1, ext2], compiler_directives={'embedsignature': True}),
  ...
)

Makefile:

MKDIR          := mkdir -p
LN             := ln -fs
RM             := rm
CP             := cp
CD             := cd
COMPILER       := gfortran

GENERAL_FLAGS            := -std=f2003 -fall-intrinsics -fPIC -cpp -static
RELEASE_FLAGS            := ${GENERAL_FLAGS} -Ofast

SRC_DIR                      := src/model
BUILD_DIR                    := build
RELEASE_DIR                  := release
DIST_DIR                     := dist

PROGRAM_NAME              := model

FILES := Utils.o Logger.o ...

.PHONY: release

debug: FLAGS       := ${DEBUG_FLAGS} -o
debug: OBJ_DIR     := ${BUILD_DIR}/${DEBUG_DIR}
debug: PROGRAM_DIR := ${DIST_DIR}/${DEBUG_DIR}
debug: PROGRAM     := ${PROGRAM_DIR}/${PROGRAM_NAME}
debug: COMP        := ${COMPILER}

release: FLAGS       := ${RELEASE_FLAGS} -o
release: OBJ_DIR     := ${BUILD_DIR}/${RELEASE_DIR}
release: PROGRAM_DIR := ${DIST_DIR}/${RELEASE_DIR}
release: PROGRAM     := ${PROGRAM_DIR}/${PROGRAM_NAME}
release: COMP        := ${COMPILER}

.SECONDEXPANSION:
release: $${FILES}
      ${MKDIR} ${PROGRAM_DIR}
      ${COMP} -I ${OBJ_DIR} ${FLAGS} ${PROGRAM} $(FILES:%o=${OBJ_DIR}/%o)
      ${LN} ${PROGRAM} $@-link

%.o: ${SRC_DIR}/%.f90
    ${MKDIR} ${OBJ_DIR}
    ${COMP} -J ${OBJ_DIR} -c ${FLAGS} ${OBJ_DIR}/$@ $<

Inside conda env:

nm model.cpython-38-x86_64-linux-gnu.so --undefined:

                 w __cxa_finalize@@GLIBC_2.2.5
                 U free@@GLIBC_2.2.5
                 U _gfortran_adjustl@@GFORTRAN_8
                 U _gfortran_compare_string@@GFORTRAN_8
                 U _gfortran_concat_string@@GFORTRAN_8
                 U _gfortran_exit_i4@@GFORTRAN_8
                 U _gfortran_fdate_sub@@GFORTRAN_8
                 U _gfortran_getarg_i4@@GFORTRAN_8
                 U _gfortran_iargc@@GFORTRAN_8
                 U _gfortran_select_string@@GFORTRAN_8
                 U _gfortran_st_inquire@@GFORTRAN_8
                 U _gfortran_st_open@@GFORTRAN_8
                 U _gfortran_st_read_done@@GFORTRAN_8
                 U _gfortran_st_read@@GFORTRAN_8
                 U _gfortran_string_len_trim@@GFORTRAN_8
                 U _gfortran_string_trim@@GFORTRAN_8
                 U _gfortran_st_write_done@@GFORTRAN_8
                 U _gfortran_st_write@@GFORTRAN_8
                 ...

ldd model.cpython-38-x86_64-linux-gnu.so:

        linux-vdso.so.1 (0x00007ffc07da3000)
        libgfortran.so.5 => /usr/lib64/libgfortran.so.5 (0x00007fe5c0b33000)
        libc.so.6 => /usr/lib64/libc.so.6 (0x00007fe5c0929000)
        libquadmath.so.0 => /home/rbenedetti/miniconda3/envs/mais-soja/lib/libquadmath.so.0 (0x00007fe5c08ef000)
        libm.so.6 => /usr/lib64/libm.so.6 (0x00007fe5c0813000)
        libgcc_s.so.1 => /home/rbenedetti/miniconda3/envs/mais-soja/lib/libgcc_s.so.1 (0x00007fe5c07ff000)
        /lib64/ld-linux-x86-64.so.2 (0x00007fe5c0e37000)

python setup.py build_ext --inplace:

MAKEFILE OUT
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Utils.o src/model/Utils.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Logger.o src/model/Logger.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Benchmark.o src/model/Benchmark.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/coefficients.o src/model/coefficients.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Results.o src/model/Results.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Parameters.o src/model/Parameters.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/IO.o src/model/IO.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Parser.o src/model/Parser.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Simulation.o src/model/Simulation.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Instrumentalization.o src/model/Instrumentalization.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/API.o src/model/API.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Program.o src/model/Program.f90
mkdir -p dist/release
gfortran -I build/release -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o dist/release/model build/release/Utils.o build/release/Logger.o build/release/Benchmark.o build/release/coefficients.o build/release/Results.o build/release/Parameters.o build/release/IO.o build/release/Parser.o build/release/Simulation.o build/release/Instrumentalization.o build/release/API.o build/release/Program.o
ln -fs dist/release/model release-link

MAKEFILE ERR

running build_ext
building 'model' extension
creating build/temp.linux-x86_64-3.8
creating build/temp.linux-x86_64-3.8/src
creating build/temp.linux-x86_64-3.8/src/cython
gcc -pthread -B /home/rbenedetti/miniconda3/envs/mais-soja/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -Isrc/cython -I/usr/local/include -I/home/rbenedetti/miniconda3/envs/mais-soja/include/python3.8 -c src/cython/model.c -o build/temp.linux-x86_64-3.8/src/cython/model.o -g -static -static-libgfortran
In file included from src/cython/model.c:724:
src/cython/API.h:28:1: warning: function declaration isn’t a prototype [-Wstrict-prototypes]
   28 | extern void wrapper_reset();
      | ^~~~~~
creating build/lib.linux-x86_64-3.8
gcc -pthread -shared -B /home/rbenedetti/miniconda3/envs/mais-soja/compiler_compat -L/home/rbenedetti/miniconda3/envs/mais-soja/lib -Wl,-rpath=/home/rbenedetti/miniconda3/envs/mais-soja/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.8/src/cython/model.o -lgfortran -o build/lib.linux-x86_64-3.8/model.cpython-38-x86_64-linux-gnu.so build/release/Utils.o build/release/Logger.o build/release/Benchmark.o build/release/coefficients.o build/release/Results.o build/release/Parameters.o build/release/IO.o build/release/Parser.o build/release/Simulation.o build/release/Instrumentalization.o build/release/API.o
copying build/lib.linux-x86_64-3.8/model.cpython-38-x86_64-linux-gnu.so ->

Outside of conda env:

nm model.cpython-38-x86_64-linux-gnu.so --undefined:

                 w __cxa_finalize@GLIBC_2.2.5
                 U free@GLIBC_2.2.5
                 U _gfortran_adjustl
                 U _gfortran_compare_string
                 U _gfortran_concat_string
                 U _gfortran_exit_i4
                 U _gfortran_fdate_sub
                 U _gfortran_getarg_i4
                 U _gfortran_iargc
                 U _gfortran_select_string
                 U _gfortran_st_inquire
                 ...

ldd model.cpython-38-x86_64-linux-gnu.so:

        linux-vdso.so.1 (0x00007ffce5559000)
        libc.so.6 => /usr/lib64/libc.so.6 (0x00007f6457e74000)
        /lib64/ld-linux-x86-64.so.2 (0x00007f64580d2000)

python setup.py build_ext --inplace:

python setup.py build_ext --inplace
MAKEFILE OUT
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Utils.o src/model/Utils.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Logger.o src/model/Logger.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Benchmark.o src/model/Benchmark.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Coefficients.o src/model/Coefficients.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Results.o src/model/Results.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Parameters.o src/model/Parameters.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/IO.o src/model/IO.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Parser.o src/model/Parser.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Simulation.o src/model/Simulation.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Instrumentalization.o src/model/Instrumentalization.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/API.o src/model/API.f90
mkdir -p build/release
gfortran -J build/release -c -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o build/release/Program.o src/model/Program.f90
mkdir -p dist/release
gfortran -I build/release -std=f2003 -fall-intrinsics -fPIC -cpp -static -static-libgfortran -Ofast -o dist/release/model build/release/Utils.o build/release/Logger.o build/release/Benchmark.o build/release/Coeficients.o build/release/Results.o build/release/Parameters.o build/release/IO.o build/release/Parser.o build/release/Simulation.o build/release/Instrumentalization.o build/release/API.o build/release/Program.o
ln -fs dist/release/model release-link

MAKEFILE ERR

running build_ext
building 'model' extension
creating build/temp.linux-x86_64-3.10
creating build/temp.linux-x86_64-3.10/src
creating build/temp.linux-x86_64-3.10/src/cython
gcc -Wno-unused-result -Wsign-compare -DDYNAMIC_ANNOTATIONS_ENABLED=1 -DNDEBUG -O2 -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -m64 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -D_GNU_SOURCE -fPIC -fwrapv -O2 -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -m64 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -D_GNU_SOURCE -fPIC -fwrapv -O2 -fexceptions -g -grecord-gcc-switches -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -m64 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -D_GNU_SOURCE -fPIC -fwrapv -fPIC -Isrc/cython -I/usr/local/include -I/usr/include/python3.10 -c src/cython/model.c -o build/temp.linux-x86_64-3.10/src/cython/model.o -g -static -static-libgfortran
creating build/lib.linux-x86_64-3.10
gcc -shared -Wl,-z,relro -Wl,--as-needed -Wl,-z,now -g -Wl,-z,relro -Wl,--as-needed -Wl,-z,now -g build/temp.linux-x86_64-3.10/src/cython/model.o -L/usr/lib64 -lgfortran -o build/lib.linux-x86_64-3.10/model.cpython-310-x86_64-linux-gnu.so build/release/Utils.o build/release/Logger.o build/release/Benchmark.o build/release/Coefficients.o build/release/Results.o build/release/Parameters.o build/release/IO.o build/release/Parser.o build/release/Simulation.o build/release/Instrumentalization.o build/release/API.o
copying build/lib.linux-x86_64-3.10/model.cpython-310-x86_64-linux-gnu.so ->

Any idea why GCC refuse to link the needed libraries outside a conda environment? As you may expect, not linking gfortran results in undefined reference when importing the module in python.

Extra system info:

find /usr -name libgfortran*:

/usr/lib/gcc/x86_64-redhat-linux/11/32/libgfortran.a
/usr/lib/gcc/x86_64-redhat-linux/11/32/libgfortran.so
/usr/lib/gcc/x86_64-redhat-linux/11/libgfortran.a
/usr/lib/gcc/x86_64-redhat-linux/11/libgfortran.so
/usr/lib/gcc/x86_64-redhat-linux/11/libgfortran.spec
/usr/lib/libgfortran.so.5
/usr/lib/libgfortran.so.5.0.0
/usr/lib/libgfortran.so
/usr/lib/libgfortran.a
/usr/lib64/python3.5/site-packages/scipy/.libs/libgfortran-ed201abd.so.3.0.0
/usr/lib64/python3.5/site-packages/numpy/.libs/libgfortran-ed201abd.so.3.0.0
/usr/lib64/libgfortran.so.1
/usr/lib64/libgfortran.so.1.0.0
/usr/lib64/libgfortran.so.5
/usr/lib64/libgfortran.so.5.0.0
...

python

linker

cython

gfortran

0 Answers

Your Answer

Accepted video resources

Posts

Questions

Blogs

Jobs