[ Jul 23 07:46:40 Enabled. ] [ Jul 23 07:46:40 Rereading configuration. ] [ Jul 23 07:46:42 Rereading configuration. ] [ Jul 23 07:46:46 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2f2dde38-7c32-463a-81ec-e9938296aff2/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_2f2dde38-7c32-463a-81ec-e9938296aff2/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:46:47.209941597Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551} {"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:46:47.212865622Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:46:47.212878972Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:46:47.22900919Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:46:47.34844978Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:52.944172984Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:39682"} {"msg":"region 5de29da7-aea4-47b5-8efa-c06595dc12d2 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:52.985623078Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:52.990011865Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"uri":"/crucible/0/regions","method":"POST","req_id":"a45e61bb-40d3-4b03-a4c7-69502565d93d","remote_addr":"[fd00:1122:3344:101::c]:39682","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":23112,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:52.992360826Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_2f2dde38-7c32-463a-81ec-e9938296aff2/crucible/regions/5de29da7-aea4-47b5-8efa-c06595dc12d2","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.005402547Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_2f2dde38-7c32-463a-81ec-e9938296aff2/crucible/regions/5de29da7-aea4-47b5-8efa-c06595dc12d2","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.005434727Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"5de29da7-aea4-47b5-8efa-c06595dc12d2\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/5de29da7-aea4-47b5-8efa-c06595dc12d2\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.064637683Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"region":"5de29da7-aea4-47b5-8efa-c06595dc12d2","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"5de29da7-aea4-47b5-8efa-c06595dc12d2\" \"--data\" \"/data/regions/5de29da7-aea4-47b5-8efa-c06595dc12d2\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.066368758Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"region":"5de29da7-aea4-47b5-8efa-c06595dc12d2","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.239103236Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"uri":"/crucible/0/regions","method":"POST","req_id":"c2273ef2-9edf-4c71-b380-f2361bab537e","remote_addr":"[fd00:1122:3344:101::c]:39682","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":67,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.386741675Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"region":"5de29da7-aea4-47b5-8efa-c06595dc12d2","component":"worker"} {"msg":"region 5de29da7-aea4-47b5-8efa-c06595dc12d2 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.386798146Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.386897507Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-5de29da7-aea4-47b5-8efa-c06595dc12d2","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.38732385Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-5de29da7-aea4-47b5-8efa-c06595dc12d2","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.397083594Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.419262203Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-5de29da7-aea4-47b5-8efa-c06595dc12d2","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.422000356Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/5de29da7-aea4-47b5-8efa-c06595dc12d2","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.422746393Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.422804133Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.422844784Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.422880244Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.426663476Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.444307807Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:53.894824626Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"uri":"/crucible/0/regions","method":"POST","req_id":"7da09c2a-b919-4037-a16b-8f32cef5513e","remote_addr":"[fd00:1122:3344:101::c]:39682","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":50,"response_code":"200"} {"msg":"region bedd1856-dbff-46c0-84b9-9bab179104f3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.535334518Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.535531079Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"uri":"/crucible/0/regions","method":"POST","req_id":"7fb63d5a-95cd-4d73-80a6-22212a604bb5","remote_addr":"[fd00:1122:3344:101::c]:39682","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":240,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.535550969Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_2f2dde38-7c32-463a-81ec-e9938296aff2/crucible/regions/bedd1856-dbff-46c0-84b9-9bab179104f3","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.544239383Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_2f2dde38-7c32-463a-81ec-e9938296aff2/crucible/regions/bedd1856-dbff-46c0-84b9-9bab179104f3","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.544275284Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"bedd1856-dbff-46c0-84b9-9bab179104f3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/bedd1856-dbff-46c0-84b9-9bab179104f3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.604385137Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"region":"bedd1856-dbff-46c0-84b9-9bab179104f3","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"bedd1856-dbff-46c0-84b9-9bab179104f3\" \"--data\" \"/data/regions/bedd1856-dbff-46c0-84b9-9bab179104f3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.604446678Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"region":"bedd1856-dbff-46c0-84b9-9bab179104f3","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.687488578Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"region":"bedd1856-dbff-46c0-84b9-9bab179104f3","component":"worker"} {"msg":"region bedd1856-dbff-46c0-84b9-9bab179104f3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.687534308Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.687619799Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-bedd1856-dbff-46c0-84b9-9bab179104f3","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.689154682Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-bedd1856-dbff-46c0-84b9-9bab179104f3","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.692348009Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.699835283Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-bedd1856-dbff-46c0-84b9-9bab179104f3","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.702863739Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/bedd1856-dbff-46c0-84b9-9bab179104f3","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.703091341Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.703154861Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.703204772Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.703257172Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.707016264Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.711421912Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:57:59.743814229Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"uri":"/crucible/0/regions","method":"POST","req_id":"76ce9d91-5570-461d-b909-0081cb9524c9","remote_addr":"[fd00:1122:3344:101::c]:39682","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":57,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.521656216Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"uri":"/crucible/0/regions/bedd1856-dbff-46c0-84b9-9bab179104f3","method":"GET","req_id":"940b4066-4a58-4876-aeec-79b386cde3f3","remote_addr":"[fd00:1122:3344:101::c]:39682","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":89,"response_code":"200"} {"msg":"path is \"/data/regions/bedd1856-dbff-46c0-84b9-9bab179104f3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.52557358Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"datafile"} {"msg":"dataset is oxp_2f2dde38-7c32-463a-81ec-e9938296aff2/crucible/regions/bedd1856-dbff-46c0-84b9-9bab179104f3","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.537837664Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"datafile"} {"msg":"region bedd1856-dbff-46c0-84b9-9bab179104f3 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.547793429Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.548014941Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.548071702Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"uri":"/crucible/0/regions/bedd1856-dbff-46c0-84b9-9bab179104f3","method":"DELETE","req_id":"e25073a8-2c1b-4f87-b5ee-1f9d7e0fdd31","remote_addr":"[fd00:1122:3344:101::c]:39682","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":22524,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-bedd1856-dbff-46c0-84b9-9bab179104f3 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.549169411Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.551589092Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"uri":"/crucible/0/regions/bedd1856-dbff-46c0-84b9-9bab179104f3","method":"GET","req_id":"0a0d6e05-2001-4e99-861a-fd328a58811d","remote_addr":"[fd00:1122:3344:101::c]:39682","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":37,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.557784055Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_2f2dde38-7c32-463a-81ec-e9938296aff2/crucible/regions/bedd1856-dbff-46c0-84b9-9bab179104f3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.570382533Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"region":"bedd1856-dbff-46c0-84b9-9bab179104f3","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.609922Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"uri":"/crucible/0/regions/bedd1856-dbff-46c0-84b9-9bab179104f3","method":"GET","req_id":"4250311d-14a4-4a44-a066-09f2544b24e4","remote_addr":"[fd00:1122:3344:101::c]:39682","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":60,"response_code":"200"} {"msg":"region bedd1856-dbff-46c0-84b9-9bab179104f3 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.63683462Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-07-23T07:58:24.780610299Z","hostname":"oxz_crucible_d881da8e-08a1-4197-b79c-32409dc13152","pid":26551,"uri":"/crucible/0/regions/bedd1856-dbff-46c0-84b9-9bab179104f3","method":"GET","req_id":"d12960e4-85f1-4c34-8578-aac0f2f85a2e","remote_addr":"[fd00:1122:3344:101::c]:39682","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":33,"response_code":"200"}