[ Nov  1 08:19:35 Enabled. ]
[ Nov  1 08:19:35 Rereading configuration. ]
[ Nov  1 08:19:37 Rereading configuration. ]
[ Nov  1 08:19:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_8ced4573-45f7-474a-9fb1-aea34b9e8e79/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_8ced4573-45f7-474a-9fb1-aea34b9e8e79/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:19:38.898125104Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:19:38.898526873Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:19:38.898537544Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:19:38.921785975Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:19:39.026811896Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.05893523Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:50563"}
{"msg":"region a623f4be-fca5-4d0f-8c12-e7815e205683 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.099932758Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.104966132Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"uri":"/crucible/0/regions","method":"POST","req_id":"4a636188-ea3c-4f7d-91b3-202da1a976ad","remote_addr":"[fd00:1122:3344:101::b]:50563","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18016,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.107083244Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_8ced4573-45f7-474a-9fb1-aea34b9e8e79/crucible/regions/a623f4be-fca5-4d0f-8c12-e7815e205683","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.117650553Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_8ced4573-45f7-474a-9fb1-aea34b9e8e79/crucible/regions/a623f4be-fca5-4d0f-8c12-e7815e205683","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.117686144Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"a623f4be-fca5-4d0f-8c12-e7815e205683\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/a623f4be-fca5-4d0f-8c12-e7815e205683\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.180076417Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"region":"a623f4be-fca5-4d0f-8c12-e7815e205683","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"a623f4be-fca5-4d0f-8c12-e7815e205683\" \"--data\" \"/data/regions/a623f4be-fca5-4d0f-8c12-e7815e205683\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.184817934Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"region":"a623f4be-fca5-4d0f-8c12-e7815e205683","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.276853816Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"uri":"/crucible/0/regions","method":"POST","req_id":"f1bb9c09-e7bc-416f-b7fe-10371f49a1c6","remote_addr":"[fd00:1122:3344:101::b]:50563","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":92,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.471471619Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"region":"a623f4be-fca5-4d0f-8c12-e7815e205683","component":"worker"}
{"msg":"region a623f4be-fca5-4d0f-8c12-e7815e205683 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.47151925Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.471609722Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-a623f4be-fca5-4d0f-8c12-e7815e205683","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.472052603Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-a623f4be-fca5-4d0f-8c12-e7815e205683","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.477436205Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.494036264Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-a623f4be-fca5-4d0f-8c12-e7815e205683","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.496876963Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/a623f4be-fca5-4d0f-8c12-e7815e205683","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.498242387Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.500575894Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.500639766Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.500686497Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.507870573Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.512042106Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:46.974407969Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"uri":"/crucible/0/regions","method":"POST","req_id":"d878c2a6-767b-4951-92da-0cd28df58b84","remote_addr":"[fd00:1122:3344:101::b]:50563","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.268132817Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"uri":"/crucible/0/regions/a623f4be-fca5-4d0f-8c12-e7815e205683","method":"GET","req_id":"ddedf264-dd86-4525-9c35-3bcd1562ccb7","remote_addr":"[fd00:1122:3344:101::b]:50563","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":99,"response_code":"200"}
{"msg":"path is \"/data/regions/a623f4be-fca5-4d0f-8c12-e7815e205683\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.270885625Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"datafile"}
{"msg":"dataset is oxp_8ced4573-45f7-474a-9fb1-aea34b9e8e79/crucible/regions/a623f4be-fca5-4d0f-8c12-e7815e205683","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.281307991Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"datafile"}
{"msg":"region a623f4be-fca5-4d0f-8c12-e7815e205683 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.290354633Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.290529818Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"disabling downstairs instance: downstairs-a623f4be-fca5-4d0f-8c12-e7815e205683 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.291567093Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.292788103Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"uri":"/crucible/0/regions/a623f4be-fca5-4d0f-8c12-e7815e205683","method":"DELETE","req_id":"c3efeed6-9b17-4a57-a8b9-2fbd0c4b2a6b","remote_addr":"[fd00:1122:3344:101::b]:50563","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":21925,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.295841818Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"uri":"/crucible/0/regions/a623f4be-fca5-4d0f-8c12-e7815e205683","method":"GET","req_id":"dbf2b852-08df-4287-89bd-fe8776e4a2d8","remote_addr":"[fd00:1122:3344:101::b]:50563","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.29590978Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_8ced4573-45f7-474a-9fb1-aea34b9e8e79/crucible/regions/a623f4be-fca5-4d0f-8c12-e7815e205683\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.306369497Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"region":"a623f4be-fca5-4d0f-8c12-e7815e205683","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.359641456Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"uri":"/crucible/0/regions/a623f4be-fca5-4d0f-8c12-e7815e205683","method":"GET","req_id":"625cfe29-3dd1-4d68-8cbb-7182b0813060","remote_addr":"[fd00:1122:3344:101::b]:50563","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"region a623f4be-fca5-4d0f-8c12-e7815e205683 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.379088104Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-01T08:20:47.483352366Z","hostname":"oxz_crucible_70adb8a4-b4de-470e-a9cd-b7e15e8f2e47","pid":20709,"uri":"/crucible/0/regions/a623f4be-fca5-4d0f-8c12-e7815e205683","method":"GET","req_id":"02e36b16-d2af-41de-acde-90b410715f8c","remote_addr":"[fd00:1122:3344:101::b]:50563","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":23,"response_code":"200"}