SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") && echo $SITE_PACKAGES &&CUDNN_PATH="$SITE_PACKAGES/nvidia/cudnn"
CPLUS_INCLUDE_PATH="$SITE_PACKAGES/nvidia/cudnn/include:$SITE_PACKAGES/nvidia/nccl/include"
pip install --no-build-isolation transformer-engine[pytorch] --no-cache-dir
/data/hpc/software/ubuntu/x86_64/cuda/12.8.1/include/cudnn.h:60:10: fatal error: cudnn_version.h: No such file or directory
60 | #include "cudnn_version.h"
| ^~~~~~~~~~~~~~~~~
compilation terminated.
ninja: build stopped: subcommand failed.
Traceback (most recent call last):
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/urllib/request.py", line 1348, in do_open
h.request(req.get_method(), req.selector, req.data, headers,
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/http/client.py", line 1323, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/http/client.py", line 1369, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/http/client.py", line 1318, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/http/client.py", line 1078, in _send_output
self.send(msg)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/http/client.py", line 1016, in send
self.connect()
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/http/client.py", line 1495, in connect
self.sock = self._context.wrap_socket(self.sock,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/ssl.py", line 517, in wrap_socket
return self.sslsocket_class._create(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/ssl.py", line 1104, in _create
self.do_handshake()
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/ssl.py", line 1382, in do_handshake
self._sslobj.do_handshake()
ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'github.com'. (_ssl.c:1016)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<string>", line 128, in run
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/urllib/request.py", line 241, in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/urllib/request.py", line 216, in urlopen
return opener.open(url, data, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/urllib/request.py", line 519, in open
response = self._open(req, data)
^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/urllib/request.py", line 536, in _open
result = self._call_chain(self.handle_open, protocol, protocol +
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/urllib/request.py", line 496, in _call_chain
result = func(*args)
^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/urllib/request.py", line 1391, in https_open
return self.do_open(http.client.HTTPSConnection, req,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/urllib/request.py", line 1351, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: Hostname mismatch, certificate is not valid for 'github.com'. (_ssl.c:1016)>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/torch/utils/cpp_extension.py", line 2693, in _run_ninja_build
subprocess.run(
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '['ninja', '-v']' returned non-zero exit status 1.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 389, in <module>
main()
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 373, in main
json_out["return_val"] = hook(**hook_input["kwargs"])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 280, in build_wheel
return _build_backend().build_wheel(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/build_meta.py", line 439, in build_wheel
return _build(['bdist_wheel', '--dist-info-dir', str(metadata_directory)])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/build_meta.py", line 427, in _build
return self._build_with_temp_dir(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/build_meta.py", line 408, in _build_with_temp_dir
self.run_setup()
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/build_meta.py", line 518, in run_setup
super().run_setup(setup_script=setup_script)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/build_meta.py", line 317, in run_setup
exec(code, locals())
File "<string>", line 171, in <module>
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/__init__.py", line 117, in setup
return distutils.core.setup(**attrs) # type: ignore[return-value]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/core.py", line 186, in setup
return run_commands(dist)
^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/core.py", line 202, in run_commands
dist.run_commands()
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/dist.py", line 1002, in run_commands
self.run_command(cmd)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/dist.py", line 1107, in run_command
super().run_command(command)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/dist.py", line 1021, in run_command
cmd_obj.run()
File "<string>", line 146, in run
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/command/bdist_wheel.py", line 370, in run
self.run_command("build")
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/cmd.py", line 357, in run_command
self.distribution.run_command(command)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/dist.py", line 1107, in run_command
super().run_command(command)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/dist.py", line 1021, in run_command
cmd_obj.run()
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/command/build.py", line 135, in run
self.run_command(cmd_name)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/cmd.py", line 357, in run_command
self.distribution.run_command(command)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/dist.py", line 1107, in run_command
super().run_command(command)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/dist.py", line 1021, in run_command
cmd_obj.run()
File "/home/singularity/tmp/pip-install-k2u6_8c5/transformer-engine-torch_da0f21ebf1cd4ee5a45f1b87127159af/build_tools/build_ext.py", line 137, in run
super().run()
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/command/build_ext.py", line 97, in run
_build_ext.run(self)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/command/build_ext.py", line 368, in run
self.build_extensions()
File "/home/singularity/tmp/pip-install-k2u6_8c5/transformer-engine-torch_da0f21ebf1cd4ee5a45f1b87127159af/build_tools/build_ext.py", line 214, in build_extensions
super().build_extensions()
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/torch/utils/cpp_extension.py", line 1143, in build_extensions
build_ext.build_extensions(self)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/command/build_ext.py", line 484, in build_extensions
self._build_extensions_serial()
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/command/build_ext.py", line 510, in _build_extensions_serial
self.build_extension(ext)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/command/build_ext.py", line 262, in build_extension
_build_ext.build_extension(self, ext)
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/setuptools/_distutils/command/build_ext.py", line 565, in build_extension
objects = self.compiler.compile(
^^^^^^^^^^^^^^^^^^^^^^
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/torch/utils/cpp_extension.py", line 900, in unix_wrap_ninja_compile
_write_ninja_file_and_compile_objects(
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/torch/utils/cpp_extension.py", line 2297, in _write_ninja_file_and_compile_objects
_run_ninja_build(
File "/data/hpc/home/linchengxi.zeng/.conda/envs/swift/lib/python3.11/site-packages/torch/utils/cpp_extension.py", line 2710, in _run_ninja_build
raise RuntimeError(message) from e
RuntimeError: Error compiling objects for extension
[end of output]
note: This error originates from a subprocess, and is likely not a problem with pip.
ERROR: Failed building wheel for transformer_engine_torch
Failed to build transformer_engine_torch
error: failed-wheel-build-for-install
× Failed to build installable wheels for some pyproject.toml based projects
╰─> transformer_engine_torch
SITE_PACKAGES=$(python -c "import site; print(site.getsitepackages()[0])") && echo $SITE_PACKAGES &&
CUDNN_PATH="$SITE_PACKAGES/nvidia/cudnn"
CPLUS_INCLUDE_PATH="$SITE_PACKAGES/nvidia/cudnn/include:$SITE_PACKAGES/nvidia/nccl/include"
pip install --no-build-isolation transformer-engine[pytorch] --no-cache-dir