[ Dec 21 01:16:02 Enabled. ] [ Dec 21 01:16:02 Rereading configuration. ] [ Dec 21 01:16:04 Rereading configuration. ] [ Dec 21 01:16:11 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:16:11.637644611Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593} {"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:16:11.641297023Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:16:11.641307033Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:16:11.656072402Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:16:11.759068856Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.556409414Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:63834"} {"msg":"region b2700033-df0d-4689-9eac-0104ae39e1e2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.596850854Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.603541817Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.608429039Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"uri":"/crucible/0/regions","method":"POST","req_id":"50399521-8c42-4893-8b19-d9d7a1c86582","remote_addr":"[fd00:1122:3344:101::a]:63834","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":17770,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible/regions/b2700033-df0d-4689-9eac-0104ae39e1e2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.613759909Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible/regions/b2700033-df0d-4689-9eac-0104ae39e1e2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.61380934Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"b2700033-df0d-4689-9eac-0104ae39e1e2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b2700033-df0d-4689-9eac-0104ae39e1e2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.668094754Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"region":"b2700033-df0d-4689-9eac-0104ae39e1e2","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b2700033-df0d-4689-9eac-0104ae39e1e2\" \"--data\" \"/data/regions/b2700033-df0d-4689-9eac-0104ae39e1e2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.670050397Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"region":"b2700033-df0d-4689-9eac-0104ae39e1e2","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.817846165Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"uri":"/crucible/0/regions","method":"POST","req_id":"7ce40cc7-1868-44b9-b2ae-2803984b990a","remote_addr":"[fd00:1122:3344:101::a]:63834","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":78,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.825040346Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"region":"b2700033-df0d-4689-9eac-0104ae39e1e2","component":"worker"} {"msg":"region b2700033-df0d-4689-9eac-0104ae39e1e2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.825064907Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.825213799Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-b2700033-df0d-4689-9eac-0104ae39e1e2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.825590546Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b2700033-df0d-4689-9eac-0104ae39e1e2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.830434017Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.84246089Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b2700033-df0d-4689-9eac-0104ae39e1e2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.845900948Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b2700033-df0d-4689-9eac-0104ae39e1e2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.84662329Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.846689061Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.846736922Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.846778022Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.849976766Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:23.853963433Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:17:24.331069266Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"uri":"/crucible/0/regions","method":"POST","req_id":"731d9140-fd28-47d2-911e-1daa3b381600","remote_addr":"[fd00:1122:3344:101::a]:63834","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":74,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.638027173Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:55730"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.642374156Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"uri":"/crucible/0/regions/b2700033-df0d-4689-9eac-0104ae39e1e2","method":"GET","req_id":"a4d8e19a-5f49-4327-8fe0-95ce63e79031","remote_addr":"[fd00:1122:3344:101::a]:55730","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":4291,"response_code":"200"} {"msg":"path is \"/data/regions/b2700033-df0d-4689-9eac-0104ae39e1e2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.650665596Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"datafile"} {"msg":"dataset is oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible/regions/b2700033-df0d-4689-9eac-0104ae39e1e2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.665373564Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"datafile"} {"msg":"snapshot is oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible/regions/b2700033-df0d-4689-9eac-0104ae39e1e2@f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.679496081Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible/regions/b2700033-df0d-4689-9eac-0104ae39e1e2\", \"f46300ad-91ae-40ca-bb40-7dba2e257bcc\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.679543652Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.697957382Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"uri":"/crucible/0/regions/b2700033-df0d-4689-9eac-0104ae39e1e2/snapshots/f46300ad-91ae-40ca-bb40-7dba2e257bcc","method":"GET","req_id":"83806f93-1591-46e8-adbf-9c95236d3377","remote_addr":"[fd00:1122:3344:101::a]:55730","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":47307,"response_code":"200"} {"msg":"path is \"/data/regions/b2700033-df0d-4689-9eac-0104ae39e1e2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.706202671Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"datafile"} {"msg":"dataset is oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible/regions/b2700033-df0d-4689-9eac-0104ae39e1e2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.719674298Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"datafile"} {"msg":"snapshot is oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible/regions/b2700033-df0d-4689-9eac-0104ae39e1e2@f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.730259237Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible/regions/b2700033-df0d-4689-9eac-0104ae39e1e2\", \"f46300ad-91ae-40ca-bb40-7dba2e257bcc\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.730291038Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot b2700033-df0d-4689-9eac-0104ae39e1e2-f46300ad-91ae-40ca-bb40-7dba2e257bcc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.739555393Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"datafile"} {"msg":"applying SMF actions for region b2700033-df0d-4689-9eac-0104ae39e1e2 running snapshot f46300ad-91ae-40ca-bb40-7dba2e257bcc (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.740210664Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.740263604Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"uri":"/crucible/0/regions/b2700033-df0d-4689-9eac-0104ae39e1e2/snapshots/f46300ad-91ae-40ca-bb40-7dba2e257bcc/run","method":"POST","req_id":"73ac571b-ae0d-46b8-8c40-1ca561269d44","remote_addr":"[fd00:1122:3344:101::a]:55730","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":34667,"response_code":"200"} {"msg":"creating missing snapshot instance snapshot-b2700033-df0d-4689-9eac-0104ae39e1e2-f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.749890867Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-b2700033-df0d-4689-9eac-0104ae39e1e2-f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.753600659Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-b2700033-df0d-4689-9eac-0104ae39e1e2-f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.766844962Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b2700033-df0d-4689-9eac-0104ae39e1e2/.zfs/snapshot/f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.767069216Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.767120327Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.767162028Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.767203848Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.767252839Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.771014502Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.775416476Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"worker"} {"msg":"region b2700033-df0d-4689-9eac-0104ae39e1e2 running snapshot f46300ad-91ae-40ca-bb40-7dba2e257bcc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.775435757Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"datafile"} {"msg":"path is \"/data/regions/b2700033-df0d-4689-9eac-0104ae39e1e2\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.980180244Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"datafile"} {"msg":"dataset is oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible/regions/b2700033-df0d-4689-9eac-0104ae39e1e2","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.989102624Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"datafile"} {"msg":"snapshot is oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible/regions/b2700033-df0d-4689-9eac-0104ae39e1e2@f46300ad-91ae-40ca-bb40-7dba2e257bcc","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.997538635Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_22c20cb3-3659-48b7-b12d-b9ae21224e50/crucible/regions/b2700033-df0d-4689-9eac-0104ae39e1e2\", \"f46300ad-91ae-40ca-bb40-7dba2e257bcc\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:37.997576046Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-21T01:27:38.005574071Z","hostname":"oxz_crucible_b387a3bb-a500-4bf1-a8ce-4c3de63bfe21","pid":24593,"uri":"/crucible/0/regions/b2700033-df0d-4689-9eac-0104ae39e1e2/snapshots/f46300ad-91ae-40ca-bb40-7dba2e257bcc/run","method":"POST","req_id":"87bf8b3f-a353-48d0-818e-5cb6b72fa4c3","remote_addr":"[fd00:1122:3344:101::a]:55730","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":25433,"response_code":"200"}