Hi I have been uploading fastqs to pass through a primary pipeline, the way it is usually done. But instead of running, the process gets cancelled. I’ve attached thee log below, how do I get it to run and what could be the potential causes of the issue? (P.S.: the failed string has not been a part of the submitted yaml before, nor is it present in the current sample yaml)
2023-06-28T16:16:12.756147606Z 2023-06-28 16:16:12 arvados.arvfile[2147] ERROR: Exception doing block prefetch
2023-06-28T16:16:12.756147606Z Traceback (most recent call last):
2023-06-28T16:16:12.756147606Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/arvfile.py”, line 596, in _block_prefetch_worker
2023-06-28T16:16:12.756147606Z self._keep.get(b)
2023-06-28T16:16:12.756147606Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/retry.py”, line 177, in num_retries_setter
2023-06-28T16:16:12.756147606Z return orig_func(self, *args, **kwargs)
2023-06-28T16:16:12.756147606Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1056, in get
2023-06-28T16:16:12.756147606Z return self._get_or_head(loc_s, method=“GET”, **kwargs)
2023-06-28T16:16:12.756147606Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1179, in _get_or_head
2023-06-28T16:16:12.756147606Z “[{}] {} not found”.format(request_id, loc_s), service_errors)
2023-06-28T16:16:12.756147606Z arvados.errors.NotFoundError: [req-wdq8zephlamfk5fwcodz] 08b4602ac944d715fc69a33941fa61b6+67108864+A5c7dc1ad56243554f10b0e68541da433599640b4@64aed1c5 not found: http://10.0.255.65:25107/ responded with 404 HTTP/1.1 404 Not Found
2023-06-28T16:16:12.756147606Z ; http://10.0.255.12:25107/ responded with 404 HTTP/1.1 404 Not Found
2023-06-28T16:16:12.756147606Z
2023-06-28T16:16:12.757959694Z 2023-06-28 16:16:12 arvados.arvados_fuse[2147] ERROR: Unhandled exception during FUSE operation
2023-06-28T16:16:12.757959694Z Traceback (most recent call last):
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados_fuse/init.py”, line 327, in catch_exceptions_wrapper
2023-06-28T16:16:12.757959694Z return orig_func(self, *args, **kwargs)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados_fuse/init.py”, line 644, in read
2023-06-28T16:16:12.757959694Z r = handle.obj.readfrom(off, size, self.num_retries)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados_fuse/fusefile.py”, line 66, in readfrom
2023-06-28T16:16:12.757959694Z return self.arvfile.readfrom(off, size, num_retries, exact=True)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/arvfile.py”, line 1107, in readfrom
2023-06-28T16:16:12.757959694Z block = self.parent._my_block_manager().get_block_contents(lr.locator, num_retries=num_retries, cache_only=(bool(data) and not exact))
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/arvfile.py”, line 785, in get_block_contents
2023-06-28T16:16:12.757959694Z return self._keep.get(locator, num_retries=num_retries)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/retry.py”, line 177, in num_retries_setter
2023-06-28T16:16:12.757959694Z return orig_func(self, *args, **kwargs)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1056, in get
2023-06-28T16:16:12.757959694Z return self._get_or_head(loc_s, method=“GET”, **kwargs)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1101, in _get_or_head
2023-06-28T16:16:12.757959694Z “failed to read {}”.format(loc_s))
2023-06-28T16:16:12.757959694Z arvados.errors.KeepReadError: failed to read 08b4602ac944d715fc69a33941fa61b6+67108864+A5c7dc1ad56243554f10b0e68541da433599640b4@64aed1c5
2023-06-28T16:16:12.757959694Z 2023-06-28 16:16:12 arvados.arvados_fuse[2147] ERROR: Unhandled exception during FUSE operation
2023-06-28T16:16:12.757959694Z Traceback (most recent call last):
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados_fuse/init.py”, line 327, in catch_exceptions_wrapper
2023-06-28T16:16:12.757959694Z return orig_func(self, *args, **kwargs)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados_fuse/init.py”, line 644, in read
2023-06-28T16:16:12.757959694Z r = handle.obj.readfrom(off, size, self.num_retries)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados_fuse/fusefile.py”, line 66, in readfrom
2023-06-28T16:16:12.757959694Z return self.arvfile.readfrom(off, size, num_retries, exact=True)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/arvfile.py”, line 1107, in readfrom
2023-06-28T16:16:12.757959694Z block = self.parent._my_block_manager().get_block_contents(lr.locator, num_retries=num_retries, cache_only=(bool(data) and not exact))
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/arvfile.py”, line 785, in get_block_contents
2023-06-28T16:16:12.757959694Z return self._keep.get(locator, num_retries=num_retries)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/retry.py”, line 177, in num_retries_setter
2023-06-28T16:16:12.757959694Z return orig_func(self, *args, **kwargs)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1056, in get
2023-06-28T16:16:12.757959694Z return self._get_or_head(loc_s, method=“GET”, **kwargs)
2023-06-28T16:16:12.757959694Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1101, in _get_or_head
2023-06-28T16:16:12.757959694Z “failed to read {}”.format(loc_s))
2023-06-28T16:16:12.757959694Z arvados.errors.KeepReadError: failed to read 08b4602ac944d715fc69a33941fa61b6+67108864+A5c7dc1ad56243554f10b0e68541da433599640b4@64aed1c5
2023-06-28T16:16:12.920281521Z 2023-06-28 16:16:12 arvados.arvfile[2147] ERROR: Exception doing block prefetch
2023-06-28T16:16:12.920281521Z Traceback (most recent call last):
2023-06-28T16:16:12.920281521Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/arvfile.py”, line 596, in _block_prefetch_worker
2023-06-28T16:16:12.920281521Z self._keep.get(b)
2023-06-28T16:16:12.920281521Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/retry.py”, line 177, in num_retries_setter
2023-06-28T16:16:12.920281521Z return orig_func(self, *args, **kwargs)
2023-06-28T16:16:12.920281521Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1056, in get
2023-06-28T16:16:12.920281521Z return self._get_or_head(loc_s, method=“GET”, **kwargs)
2023-06-28T16:16:12.920281521Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1101, in _get_or_head
2023-06-28T16:16:12.920281521Z “failed to read {}”.format(loc_s))
2023-06-28T16:16:12.920281521Z arvados.errors.KeepReadError: failed to read 08b4602ac944d715fc69a33941fa61b6+67108864+A5c7dc1ad56243554f10b0e68541da433599640b4@64aed1c5
2023-06-28T16:16:16.800414958Z crunchstat: keepcalls 0 put 2586 get – interval 10.0000 seconds 0 put 2586 get
2023-06-28T16:16:16.800414958Z crunchstat: net:keep0 0 tx 201326592 rx – interval 10.0000 seconds 0 tx 201326592 rx
2023-06-28T16:16:16.800414958Z crunchstat: keepcache 2581 hit 5 miss – interval 10.0000 seconds 2581 hit 5 miss
2023-06-28T16:16:16.800414958Z crunchstat: blkio:0:0 0 write 201326592 read – interval 10.0000 seconds 0 write 201326592 read
2023-06-28T16:16:16.800414958Z crunchstat: fuseops 0 write 1555 read – interval 10.0000 seconds 0 write 1555 read
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:read 1554 count 12.972205 time – interval 10.0000 seconds 1554 count 12.972205 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:write 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:destroy 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:on_event 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:getattr 11 count 0.000269 time – interval 10.0000 seconds 11 count 0.000269 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:setattr 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:lookup 8 count 0.057313 time – interval 10.0000 seconds 8 count 0.057313 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:forget 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:open 1 count 0.000037 time – interval 10.0000 seconds 1 count 0.000037 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:release 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:opendir 1 count 0.000037 time – interval 10.0000 seconds 1 count 0.000037 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:readdir 2 count 0.000004 time – interval 10.0000 seconds 2 count 0.000004 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:statfs 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:create 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:mkdir 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:unlink 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:rmdir 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:rename 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z crunchstat: fuseop:flush 0 count 0.000000 time – interval 10.0000 seconds 0 count 0.000000 time
2023-06-28T16:16:16.800414958Z 2023-06-28 16:16:16 arvados.arvfile[2147] ERROR: Exception doing block prefetch
2023-06-28T16:16:16.800414958Z Traceback (most recent call last):
2023-06-28T16:16:16.800414958Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/arvfile.py”, line 596, in _block_prefetch_worker
2023-06-28T16:16:16.800414958Z self._keep.get(b)
2023-06-28T16:16:16.800414958Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/retry.py”, line 177, in num_retries_setter
2023-06-28T16:16:16.800414958Z return orig_func(self, *args, **kwargs)
2023-06-28T16:16:16.800414958Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1056, in get
2023-06-28T16:16:16.800414958Z return self._get_or_head(loc_s, method=“GET”, **kwargs)
2023-06-28T16:16:16.800414958Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1179, in _get_or_head
2023-06-28T16:16:16.800414958Z “[{}] {} not found”.format(request_id, loc_s), service_errors)
2023-06-28T16:16:16.800414958Z arvados.errors.NotFoundError: [req-jset482035npa20jhcq1] 08b4602ac944d715fc69a33941fa61b6+67108864+A5c7dc1ad56243554f10b0e68541da433599640b4@64aed1c5 not found: http://10.0.255.65:25107/ responded with 404 HTTP/1.1 404 Not Found
2023-06-28T16:16:16.800414958Z ; http://10.0.255.12:25107/ responded with 404 HTTP/1.1 404 Not Found
2023-06-28T16:16:16.800414958Z
2023-06-28T16:16:16.802186512Z 2023-06-28 16:16:16 arvados.arvfile[2147] ERROR: Exception doing block prefetch
2023-06-28T16:16:16.802186512Z Traceback (most recent call last):
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/arvfile.py”, line 596, in _block_prefetch_worker
2023-06-28T16:16:16.802186512Z self._keep.get(b)
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/retry.py”, line 177, in num_retries_setter
2023-06-28T16:16:16.802186512Z return orig_func(self, *args, **kwargs)
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1056, in get
2023-06-28T16:16:16.802186512Z return self._get_or_head(loc_s, method=“GET”, **kwargs)
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1101, in _get_or_head
2023-06-28T16:16:16.802186512Z “failed to read {}”.format(loc_s))
2023-06-28T16:16:16.802186512Z arvados.errors.KeepReadError: failed to read 08b4602ac944d715fc69a33941fa61b6+67108864+A5c7dc1ad56243554f10b0e68541da433599640b4@64aed1c5
2023-06-28T16:16:16.802186512Z 2023-06-28 16:16:16 arvados.arvados_fuse[2147] ERROR: Unhandled exception during FUSE operation
2023-06-28T16:16:16.802186512Z Traceback (most recent call last):
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados_fuse/init.py”, line 327, in catch_exceptions_wrapper
2023-06-28T16:16:16.802186512Z return orig_func(self, *args, **kwargs)
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados_fuse/init.py”, line 644, in read
2023-06-28T16:16:16.802186512Z r = handle.obj.readfrom(off, size, self.num_retries)
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados_fuse/fusefile.py”, line 66, in readfrom
2023-06-28T16:16:16.802186512Z return self.arvfile.readfrom(off, size, num_retries, exact=True)
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/arvfile.py”, line 1107, in readfrom
2023-06-28T16:16:16.802186512Z block = self.parent._my_block_manager().get_block_contents(lr.locator, num_retries=num_retries, cache_only=(bool(data) and not exact))
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/arvfile.py”, line 785, in get_block_contents
2023-06-28T16:16:16.802186512Z return self._keep.get(locator, num_retries=num_retries)
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/retry.py”, line 177, in num_retries_setter
2023-06-28T16:16:16.802186512Z return orig_func(self, *args, **kwargs)
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1056, in get
2023-06-28T16:16:16.802186512Z return self._get_or_head(loc_s, method=“GET”, **kwargs)
2023-06-28T16:16:16.802186512Z File “/usr/share/python3/dist/python3-arvados-fuse/lib/python3.7/site-packages/arvados/keep.py”, line 1101, in _get_or_head
2023-06-28T16:16:16.802186512Z “failed to read {}”.format(loc_s))
2023-06-28T16:16:16.802186512Z arvados.errors.KeepReadError: failed to read 08b4602ac944d715fc69a33941fa61b6+67108864+A5c7dc1ad56243554f10b0e68541da433599640b4@64aed1c5