[ Oct  9 23:15:58 Enabled. ]
[ Oct  9 23:15:58 Rereading configuration. ]
[ Oct  9 23:16:01 Rereading configuration. ]
[ Oct  9 23:16:04 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_0f1674be-fe99-41ac-8e3d-cac6ad2b4244/crucible -l [fd00:1122:3344:101::15]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_0f1674be-fe99-41ac-8e3d-cac6ad2b4244/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:04.27445792Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112}
{"msg":"listen IP: [fd00:1122:3344:101::15]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:04.276182254Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:04.276192114Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:04.291598539Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:04.438320511Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.406385813Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:58628"}
{"msg":"region c801f119-f695-4456-b198-9943479b2084 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.432728504Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.436706793Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"uri":"/crucible/0/regions","method":"POST","req_id":"b24cb947-f741-4fc1-a1b8-749269c279b0","remote_addr":"[fd00:1122:3344:101::b]:58628","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":9724,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.437788704Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_0f1674be-fe99-41ac-8e3d-cac6ad2b4244/crucible/regions/c801f119-f695-4456-b198-9943479b2084","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.447347663Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_0f1674be-fe99-41ac-8e3d-cac6ad2b4244/crucible/regions/c801f119-f695-4456-b198-9943479b2084","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.447375824Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"c801f119-f695-4456-b198-9943479b2084\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/c801f119-f695-4456-b198-9943479b2084\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.514916219Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"region":"c801f119-f695-4456-b198-9943479b2084","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"c801f119-f695-4456-b198-9943479b2084\" \"--data\" \"/data/regions/c801f119-f695-4456-b198-9943479b2084\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.519215624Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"region":"c801f119-f695-4456-b198-9943479b2084","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.647297238Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"region":"c801f119-f695-4456-b198-9943479b2084","component":"worker"}
{"msg":"region c801f119-f695-4456-b198-9943479b2084 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.647346159Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.647448761Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-c801f119-f695-4456-b198-9943479b2084","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.649833218Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-c801f119-f695-4456-b198-9943479b2084","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.655849637Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.675686379Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-c801f119-f695-4456-b198-9943479b2084","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.67874822Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/c801f119-f695-4456-b198-9943479b2084","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.684737118Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.684808669Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::15","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.684870261Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.684917471Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.687909771Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"uri":"/crucible/0/regions","method":"POST","req_id":"ae9c5294-d3fe-41d9-9690-b389664b5c19","remote_addr":"[fd00:1122:3344:101::b]:58628","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":72,"response_code":"200"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.689609654Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:56.69598879Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.01654057Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"uri":"/crucible/0/regions/c801f119-f695-4456-b198-9943479b2084","method":"GET","req_id":"a00e4c47-025b-4f2c-b030-705e69a82387","remote_addr":"[fd00:1122:3344:101::b]:58628","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":94,"response_code":"200"}
{"msg":"path is \"/data/regions/c801f119-f695-4456-b198-9943479b2084\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.02053242Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"datafile"}
{"msg":"dataset is oxp_0f1674be-fe99-41ac-8e3d-cac6ad2b4244/crucible/regions/c801f119-f695-4456-b198-9943479b2084","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.030858064Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"datafile"}
{"msg":"region c801f119-f695-4456-b198-9943479b2084 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.040394532Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.040547625Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.040626197Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"uri":"/crucible/0/regions/c801f119-f695-4456-b198-9943479b2084","method":"DELETE","req_id":"32cb56b0-362c-4b37-9c9d-873b6b107ce3","remote_addr":"[fd00:1122:3344:101::b]:58628","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":20122,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-c801f119-f695-4456-b198-9943479b2084 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.041587686Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.043419962Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"uri":"/crucible/0/regions/c801f119-f695-4456-b198-9943479b2084","method":"GET","req_id":"62c7f2b6-3dcb-4a9f-9464-2f63c0167b0c","remote_addr":"[fd00:1122:3344:101::b]:58628","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":37,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.046593795Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_0f1674be-fe99-41ac-8e3d-cac6ad2b4244/crucible/regions/c801f119-f695-4456-b198-9943479b2084\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.056635783Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"region":"c801f119-f695-4456-b198-9943479b2084","component":"worker"}
{"msg":"region c801f119-f695-4456-b198-9943479b2084 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.128343182Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-09T23:16:57.135879791Z","hostname":"oxz_crucible_4977ea60-05fd-4cd5-b6dc-758bfcf15337","pid":24112,"uri":"/crucible/0/regions/c801f119-f695-4456-b198-9943479b2084","method":"GET","req_id":"b6d08ce6-eb67-421a-91e4-e8fe51fb1711","remote_addr":"[fd00:1122:3344:101::b]:58628","local_addr":"[fd00:1122:3344:101::15]:32345","component":"dropshot","latency_us":546,"response_code":"200"}