[ Oct 12 03:53:12 Enabled. ]
[ Oct 12 03:53:12 Rereading configuration. ]
[ Oct 12 03:53:15 Rereading configuration. ]
[ Oct 12 03:53:17 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_42090923-c06f-4776-b57f-20fba0911cf3/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_42090923-c06f-4776-b57f-20fba0911cf3/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:17.097743945Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217}
{"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:17.10137973Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:17.10139247Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:17.121592101Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:17.20284161Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.582500574Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:33627"}
{"msg":"region c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.586375022Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.589786005Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"uri":"/crucible/0/regions","method":"POST","req_id":"7cf8674d-a16a-423d-a905-5bf60c41581f","remote_addr":"[fd00:1122:3344:101::c]:33627","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":3770,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.589807985Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_42090923-c06f-4776-b57f-20fba0911cf3/crucible/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.598134659Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_42090923-c06f-4776-b57f-20fba0911cf3/crucible/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.598170309Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.652300871Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"region":"c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0\" \"--data\" \"/data/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.652371912Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"region":"c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.796919236Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"region":"c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","component":"worker"}
{"msg":"region c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.796976717Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.797042298Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.797534984Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.800181687Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"uri":"/crucible/0/regions","method":"POST","req_id":"cad06ab6-4187-4d41-9b32-793f021cda1c","remote_addr":"[fd00:1122:3344:101::c]:33627","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":63,"response_code":"200"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.801678285Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.813264199Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.816878604Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.817662214Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.817732874Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.817786995Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.817827046Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.822116149Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:39.826590924Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.308433786Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"uri":"/crucible/0/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","method":"GET","req_id":"80d77997-4730-4297-a645-61d0ea3f7432","remote_addr":"[fd00:1122:3344:101::c]:33627","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":85,"response_code":"200"}
{"msg":"path is \"/data/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.311843018Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"datafile"}
{"msg":"dataset is oxp_42090923-c06f-4776-b57f-20fba0911cf3/crucible/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.322972876Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"datafile"}
{"msg":"region c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.332761198Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.332906009Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.33298135Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"uri":"/crucible/0/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","method":"DELETE","req_id":"9eb6e737-9339-4d4b-b24b-dfca04733330","remote_addr":"[fd00:1122:3344:101::c]:33627","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":21458,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.333949142Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.336030398Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"uri":"/crucible/0/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","method":"GET","req_id":"6d93bdc6-2922-4408-adc9-8a34804dc88e","remote_addr":"[fd00:1122:3344:101::c]:33627","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":38,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.338665421Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_42090923-c06f-4776-b57f-20fba0911cf3/crucible/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.350380576Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"region":"c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","component":"worker"}
{"msg":"region c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.418119767Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-12T03:53:40.489807757Z","hostname":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","pid":23217,"uri":"/crucible/0/regions/c8a08ddd-ee9e-4605-87f3-50b81f6fd0b0","method":"GET","req_id":"4ab91a4b-b55b-4c0e-b270-acba08e30242","remote_addr":"[fd00:1122:3344:101::c]:33627","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":72,"response_code":"200"}