[ Mar 25 05:53:41 Enabled. ]
[ Mar 25 05:53:41 Rereading configuration. ]
[ Mar 25 05:53:46 Rereading configuration. ]
[ Mar 25 05:53:54 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_7abced3b-dc0d-4ffa-8875-0a3eb3b4d372/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_7abced3b-dc0d-4ffa-8875-0a3eb3b4d372/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:53:55.060310893Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:53:55.063387911Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:53:55.063395641Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:53:55.079398212Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:53:55.222491192Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:28.781869112Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44180"}
{"msg":"region 09a759cf-2bb3-4430-84db-07c54a40797c state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:28.826164517Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:28.828718398Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:28.832425046Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"uri":"/crucible/0/regions","method":"POST","req_id":"5fa7fec0-3817-4389-94ed-fbe6758b5fca","remote_addr":"[fd00:1122:3344:101::a]:44180","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":22050,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_7abced3b-dc0d-4ffa-8875-0a3eb3b4d372/crucible/regions/09a759cf-2bb3-4430-84db-07c54a40797c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:28.839018067Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_7abced3b-dc0d-4ffa-8875-0a3eb3b4d372/crucible/regions/09a759cf-2bb3-4430-84db-07c54a40797c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:28.839044308Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"09a759cf-2bb3-4430-84db-07c54a40797c\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/09a759cf-2bb3-4430-84db-07c54a40797c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:28.898307241Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"region":"09a759cf-2bb3-4430-84db-07c54a40797c","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"09a759cf-2bb3-4430-84db-07c54a40797c\" \"--data\" \"/data/regions/09a759cf-2bb3-4430-84db-07c54a40797c\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:28.900704607Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"region":"09a759cf-2bb3-4430-84db-07c54a40797c","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.024962476Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"region":"09a759cf-2bb3-4430-84db-07c54a40797c","component":"worker"}
{"msg":"region 09a759cf-2bb3-4430-84db-07c54a40797c state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.025013968Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.02509625Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-09a759cf-2bb3-4430-84db-07c54a40797c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.025494033Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-09a759cf-2bb3-4430-84db-07c54a40797c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.030233735Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.042456155Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-09a759cf-2bb3-4430-84db-07c54a40797c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.045143771Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/09a759cf-2bb3-4430-84db-07c54a40797c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.045866714Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.045925786Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.045966997Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.046006578Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.049539701Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.060867843Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.112830013Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"uri":"/crucible/0/regions","method":"POST","req_id":"3c0fa971-ca63-44dd-ac08-7638bb27a28d","remote_addr":"[fd00:1122:3344:101::a]:44180","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":68,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.793635037Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"uri":"/crucible/0/regions/09a759cf-2bb3-4430-84db-07c54a40797c","method":"GET","req_id":"517c0440-1653-4102-8454-0bdc7ced9855","remote_addr":"[fd00:1122:3344:101::a]:44180","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":2354,"response_code":"200"}
{"msg":"path is \"/data/regions/09a759cf-2bb3-4430-84db-07c54a40797c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.796641733Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"datafile"}
{"msg":"dataset is oxp_7abced3b-dc0d-4ffa-8875-0a3eb3b4d372/crucible/regions/09a759cf-2bb3-4430-84db-07c54a40797c","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.808555973Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"datafile"}
{"msg":"region 09a759cf-2bb3-4430-84db-07c54a40797c state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.81848542Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.818624265Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.818690777Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"uri":"/crucible/0/regions/09a759cf-2bb3-4430-84db-07c54a40797c","method":"DELETE","req_id":"5969eebe-0ec9-439a-97b4-9cfcd029b12d","remote_addr":"[fd00:1122:3344:101::a]:44180","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":22069,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-09a759cf-2bb3-4430-84db-07c54a40797c (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.819816753Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.821870808Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"uri":"/crucible/0/regions/09a759cf-2bb3-4430-84db-07c54a40797c","method":"GET","req_id":"0ed24cc2-d311-47b9-a73f-ad4ecf20f2db","remote_addr":"[fd00:1122:3344:101::a]:44180","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":27,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.824785062Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_7abced3b-dc0d-4ffa-8875-0a3eb3b4d372/crucible/regions/09a759cf-2bb3-4430-84db-07c54a40797c\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.836159715Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"region":"09a759cf-2bb3-4430-84db-07c54a40797c","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.891169872Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"uri":"/crucible/0/regions/09a759cf-2bb3-4430-84db-07c54a40797c","method":"GET","req_id":"2c4d0400-a186-42ce-87ad-92f2d0b87a2b","remote_addr":"[fd00:1122:3344:101::a]:44180","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"region 09a759cf-2bb3-4430-84db-07c54a40797c state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:29.906516922Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-25T05:54:30.003889832Z","hostname":"oxz_crucible_ad4f4e79-04df-4075-9b42-22e6a81e6ade","pid":23469,"uri":"/crucible/0/regions/09a759cf-2bb3-4430-84db-07c54a40797c","method":"GET","req_id":"38a0295c-6ec6-45b2-908f-46993a9e1f56","remote_addr":"[fd00:1122:3344:101::a]:44180","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":47,"response_code":"200"}