[ Oct 15 18:33:47 Enabled. ]
[ Oct 15 18:33:47 Rereading configuration. ]
[ Oct 15 18:33:49 Rereading configuration. ]
[ Oct 15 18:33:50 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible -l [fd00:1122:3344:101::17]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:50.718793719Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118}
{"msg":"listen IP: [fd00:1122:3344:101::17]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:50.724361153Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:50.724377263Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:50.745870388Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:50.833986323Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.092947574Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:48704"}
{"msg":"region d6e84208-a5b9-4095-b717-71b3b8786bb1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.133693269Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.143118757Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions","method":"POST","req_id":"191da52e-26f8-47a3-a247-19372ed16133","remote_addr":"[fd00:1122:3344:101::c]:48704","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":18467,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.143150627Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.152971839Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.15300756Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"d6e84208-a5b9-4095-b717-71b3b8786bb1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.212974894Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"region":"d6e84208-a5b9-4095-b717-71b3b8786bb1","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"d6e84208-a5b9-4095-b717-71b3b8786bb1\" \"--data\" \"/data/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.216789637Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"region":"d6e84208-a5b9-4095-b717-71b3b8786bb1","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.45377117Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions","method":"POST","req_id":"57bc484e-7a1c-4ac0-b1fb-8df1a0d449ab","remote_addr":"[fd00:1122:3344:101::c]:48704","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":864,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.51247249Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"region":"d6e84208-a5b9-4095-b717-71b3b8786bb1","component":"worker"}
{"msg":"region d6e84208-a5b9-4095-b717-71b3b8786bb1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.51251966Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.512601112Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-d6e84208-a5b9-4095-b717-71b3b8786bb1","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.514904418Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-d6e84208-a5b9-4095-b717-71b3b8786bb1","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.522441234Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.537525016Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-d6e84208-a5b9-4095-b717-71b3b8786bb1","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.540236357Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.547839154Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.547897044Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.547935864Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.547974245Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.552486107Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.558125721Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.940186239Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions","method":"POST","req_id":"6dd52982-623f-4677-9f6c-b35b47df9db6","remote_addr":"[fd00:1122:3344:101::c]:48704","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":65,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.431923848Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1","method":"GET","req_id":"5dccd6b5-8f69-49c3-ad17-5b629de92f50","remote_addr":"[fd00:1122:3344:101::c]:48704","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"path is \"/data/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.435507879Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"dataset is oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.447481476Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"region d6e84208-a5b9-4095-b717-71b3b8786bb1 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.458100457Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.458231698Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.458316589Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1","method":"DELETE","req_id":"1559cc23-0202-4619-98aa-dff3e6cee0d9","remote_addr":"[fd00:1122:3344:101::c]:48704","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":22838,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-d6e84208-a5b9-4095-b717-71b3b8786bb1 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.459531083Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.461179812Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1","method":"GET","req_id":"f70c5c18-9e15-4505-b248-2ce1d06de51a","remote_addr":"[fd00:1122:3344:101::c]:48704","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":38,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.465674503Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.478694772Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"region":"d6e84208-a5b9-4095-b717-71b3b8786bb1","component":"worker"}
{"msg":"region d6e84208-a5b9-4095-b717-71b3b8786bb1 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.537446812Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.611175243Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions/d6e84208-a5b9-4095-b717-71b3b8786bb1","method":"GET","req_id":"be182166-1cc8-42bf-bed2-34620e0d4119","remote_addr":"[fd00:1122:3344:101::c]:48704","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":673,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:32.765275601Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:64447"}
{"msg":"region b1eef669-032f-46cc-9786-442d12f8ae27 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:32.765405203Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:32.768015693Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:32.768031513Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions","method":"POST","req_id":"c2cf8c22-268c-4fea-b239-a6ef9fcc875e","remote_addr":"[fd00:1122:3344:101::a]:64447","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":2691,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible/regions/b1eef669-032f-46cc-9786-442d12f8ae27","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:32.778872027Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible/regions/b1eef669-032f-46cc-9786-442d12f8ae27","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:32.778921277Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"b1eef669-032f-46cc-9786-442d12f8ae27\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/b1eef669-032f-46cc-9786-442d12f8ae27\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:32.890671882Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"region":"b1eef669-032f-46cc-9786-442d12f8ae27","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"b1eef669-032f-46cc-9786-442d12f8ae27\" \"--data\" \"/data/regions/b1eef669-032f-46cc-9786-442d12f8ae27\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:32.890718612Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"region":"b1eef669-032f-46cc-9786-442d12f8ae27","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:32.959828921Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions","method":"POST","req_id":"5b2579ab-3774-4133-bdc0-a015e6ff5bab","remote_addr":"[fd00:1122:3344:101::a]:64447","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.333906928Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"region":"b1eef669-032f-46cc-9786-442d12f8ae27","component":"worker"}
{"msg":"region b1eef669-032f-46cc-9786-442d12f8ae27 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.333954539Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.33407245Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-b1eef669-032f-46cc-9786-442d12f8ae27","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.335442635Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-b1eef669-032f-46cc-9786-442d12f8ae27","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.341683416Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.367800565Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-b1eef669-032f-46cc-9786-442d12f8ae27","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.375624834Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/b1eef669-032f-46cc-9786-442d12f8ae27","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.375850336Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.375893747Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.375931107Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.375967368Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.382642904Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.395855985Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:33.713273855Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions","method":"POST","req_id":"3bb26db6-ed94-4f63-b7a3-1967aca050e0","remote_addr":"[fd00:1122:3344:101::a]:64447","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":46,"response_code":"200"}
{"msg":"region 9407ffbc-e27b-462e-a669-68c452ed3456 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.742171146Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.742354088Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.742364178Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions","method":"POST","req_id":"22eec22d-e9ac-4472-8ef1-8b5918bfcb6e","remote_addr":"[fd00:1122:3344:101::a]:64447","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":944,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible/regions/9407ffbc-e27b-462e-a669-68c452ed3456","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.751756425Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible/regions/9407ffbc-e27b-462e-a669-68c452ed3456","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.751789545Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"9407ffbc-e27b-462e-a669-68c452ed3456\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9407ffbc-e27b-462e-a669-68c452ed3456\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.812602699Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"region":"9407ffbc-e27b-462e-a669-68c452ed3456","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9407ffbc-e27b-462e-a669-68c452ed3456\" \"--data\" \"/data/regions/9407ffbc-e27b-462e-a669-68c452ed3456\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.81266684Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"region":"9407ffbc-e27b-462e-a669-68c452ed3456","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.940624789Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"region":"9407ffbc-e27b-462e-a669-68c452ed3456","component":"worker"}
{"msg":"region 9407ffbc-e27b-462e-a669-68c452ed3456 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.94067579Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.940762291Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-9407ffbc-e27b-462e-a669-68c452ed3456","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.941760132Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9407ffbc-e27b-462e-a669-68c452ed3456","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.954156704Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.970166276Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.973318502Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions","method":"POST","req_id":"a8ae7123-83d8-4256-b1f1-1e9a960f9026","remote_addr":"[fd00:1122:3344:101::a]:64447","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":796,"response_code":"200"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9407ffbc-e27b-462e-a669-68c452ed3456","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.97752579Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9407ffbc-e27b-462e-a669-68c452ed3456","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.977794383Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.977841684Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::17","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.977879494Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.977922915Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.987560935Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:38:39.995466745Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.927733519Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions/9407ffbc-e27b-462e-a669-68c452ed3456","method":"GET","req_id":"788ddd6f-a889-4de8-98d3-e92b9f3a744a","remote_addr":"[fd00:1122:3344:101::a]:64447","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":69,"response_code":"200"}
{"msg":"path is \"/data/regions/9407ffbc-e27b-462e-a669-68c452ed3456\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.930329768Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"dataset is oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible/regions/9407ffbc-e27b-462e-a669-68c452ed3456","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.945638162Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"region 9407ffbc-e27b-462e-a669-68c452ed3456 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.954593554Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.954785516Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.954838987Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions/9407ffbc-e27b-462e-a669-68c452ed3456","method":"DELETE","req_id":"4a01bd7a-3585-4f6a-88d1-d99e8c5f70d2","remote_addr":"[fd00:1122:3344:101::a]:64447","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":24529,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-9407ffbc-e27b-462e-a669-68c452ed3456 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.956444765Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.958331727Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions/9407ffbc-e27b-462e-a669-68c452ed3456","method":"GET","req_id":"25cef007-30a6-48e4-852b-c3b5d0e3d254","remote_addr":"[fd00:1122:3344:101::a]:64447","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":684,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.973179286Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_a2a6c8c1-72e6-4543-bb10-d6a1d37b5f0e/crucible/regions/9407ffbc-e27b-462e-a669-68c452ed3456\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:07.983704156Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"region":"9407ffbc-e27b-462e-a669-68c452ed3456","component":"worker"}
{"msg":"region 9407ffbc-e27b-462e-a669-68c452ed3456 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:08.064317606Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:39:08.084583697Z","hostname":"oxz_crucible_67cfc8df-64be-4793-9f4a-667ce37c834c","pid":20118,"uri":"/crucible/0/regions/9407ffbc-e27b-462e-a669-68c452ed3456","method":"GET","req_id":"48b03506-d8b3-4d69-a9a1-244092d4879b","remote_addr":"[fd00:1122:3344:101::a]:64447","local_addr":"[fd00:1122:3344:101::17]:32345","component":"dropshot","latency_us":76,"response_code":"200"}