[ Nov 13 20:45:07 Enabled. ]
[ Nov 13 20:45:07 Rereading configuration. ]
[ Nov 13 20:45:08 Rereading configuration. ]
[ Nov 13 20:45:12 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:12.256410499Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574}
{"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:12.260317117Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:12.260334487Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:12.281201823Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:12.378415197Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:26.860790105Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:62674"}
{"msg":"region 14d245c6-a100-43e1-ba7e-5b05d1be50df state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:26.863233547Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:26.863512002Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions","method":"POST","req_id":"ffbdf141-ee96-4a13-aaf7-793e432396ba","remote_addr":"[fd00:1122:3344:101::b]:62674","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":455,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:26.865970815Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:26.875976701Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:26.876016091Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"14d245c6-a100-43e1-ba7e-5b05d1be50df\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:26.953856446Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"region":"14d245c6-a100-43e1-ba7e-5b05d1be50df","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"14d245c6-a100-43e1-ba7e-5b05d1be50df\" \"--data\" \"/data/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:26.953928557Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"region":"14d245c6-a100-43e1-ba7e-5b05d1be50df","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.115257655Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions","method":"POST","req_id":"23e64e59-161e-4e3e-a8c4-a3a9d6b7d5d8","remote_addr":"[fd00:1122:3344:101::b]:62674","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.156492238Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"region":"14d245c6-a100-43e1-ba7e-5b05d1be50df","component":"worker"}
{"msg":"region 14d245c6-a100-43e1-ba7e-5b05d1be50df state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.156544808Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.15664202Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-14d245c6-a100-43e1-ba7e-5b05d1be50df","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.157039337Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-14d245c6-a100-43e1-ba7e-5b05d1be50df","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.159881967Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.166626715Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-14d245c6-a100-43e1-ba7e-5b05d1be50df","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.169397314Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.169936773Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.170015475Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.170059685Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.170097796Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.17375324Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.177539176Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.382638981Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions","method":"POST","req_id":"7241729a-a384-46c5-80a4-55aaf60681b2","remote_addr":"[fd00:1122:3344:101::b]:62674","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":55,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.659379652Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df","method":"GET","req_id":"ea852aa5-40f3-430a-9b45-d8dd37b5ea11","remote_addr":"[fd00:1122:3344:101::b]:62674","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":82,"response_code":"200"}
{"msg":"path is \"/data/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.66155612Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"dataset is oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.672611924Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"region 14d245c6-a100-43e1-ba7e-5b05d1be50df state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.681977168Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.682129501Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.682205762Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df","method":"DELETE","req_id":"3b29f90b-d5aa-4f13-aaf6-202c1918bf1e","remote_addr":"[fd00:1122:3344:101::b]:62674","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":20684,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-14d245c6-a100-43e1-ba7e-5b05d1be50df (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.683334912Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.685487559Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df","method":"GET","req_id":"3b085af9-98db-47f9-ade4-9d39f913b727","remote_addr":"[fd00:1122:3344:101::b]:62674","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.68837102Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.700382671Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"region":"14d245c6-a100-43e1-ba7e-5b05d1be50df","component":"worker"}
{"msg":"region 14d245c6-a100-43e1-ba7e-5b05d1be50df state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.764636517Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:45:27.787217682Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions/14d245c6-a100-43e1-ba7e-5b05d1be50df","method":"GET","req_id":"25a1b34d-48f9-4834-9bbb-9968c4e56293","remote_addr":"[fd00:1122:3344:101::b]:62674","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.289627564Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57999"}
{"msg":"region 94abf463-bb77-4473-9ad5-5bd1b25690ee state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.289804187Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.289995651Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions","method":"POST","req_id":"ea13ae31-94b5-4b9b-9797-f129ca6956f7","remote_addr":"[fd00:1122:3344:101::a]:57999","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":220,"response_code":"200"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.290012981Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"zfs set reservation of 2684354560 for oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.298233025Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.298284426Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"94abf463-bb77-4473-9ad5-5bd1b25690ee\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.352458026Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"region":"94abf463-bb77-4473-9ad5-5bd1b25690ee","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"94abf463-bb77-4473-9ad5-5bd1b25690ee\" \"--data\" \"/data/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.354240567Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"region":"94abf463-bb77-4473-9ad5-5bd1b25690ee","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.416370376Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"region":"94abf463-bb77-4473-9ad5-5bd1b25690ee","component":"worker"}
{"msg":"region 94abf463-bb77-4473-9ad5-5bd1b25690ee state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.416404686Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.416510568Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-94abf463-bb77-4473-9ad5-5bd1b25690ee","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.417895773Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-94abf463-bb77-4473-9ad5-5bd1b25690ee","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.42229753Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.43598114Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-94abf463-bb77-4473-9ad5-5bd1b25690ee","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.43941015Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.439615083Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.439669604Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.439711415Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.439750596Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.443273568Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.447209077Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:46:21.677176007Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions","method":"POST","req_id":"ccab6ef2-24b7-49eb-b42f-e5688b52f10c","remote_addr":"[fd00:1122:3344:101::a]:57999","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":54,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:58.909284119Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44015"}
{"msg":"region ff906436-1220-4941-af9d-a7161cedb434 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:58.909437882Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:58.910547581Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:58.910560692Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions","method":"POST","req_id":"d870f1ac-eab3-43c5-80c4-de930de7ed83","remote_addr":"[fd00:1122:3344:101::a]:44015","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":1207,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/ff906436-1220-4941-af9d-a7161cedb434","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:58.921633496Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/ff906436-1220-4941-af9d-a7161cedb434","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:58.921670856Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"ff906436-1220-4941-af9d-a7161cedb434\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/ff906436-1220-4941-af9d-a7161cedb434\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:58.976810963Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"region":"ff906436-1220-4941-af9d-a7161cedb434","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"ff906436-1220-4941-af9d-a7161cedb434\" \"--data\" \"/data/regions/ff906436-1220-4941-af9d-a7161cedb434\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:58.978710046Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"region":"ff906436-1220-4941-af9d-a7161cedb434","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.042073697Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"region":"ff906436-1220-4941-af9d-a7161cedb434","component":"worker"}
{"msg":"region ff906436-1220-4941-af9d-a7161cedb434 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.042123478Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.042218169Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-ff906436-1220-4941-af9d-a7161cedb434","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.054523805Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.058297091Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions","method":"POST","req_id":"1e7f224c-4cb1-4a31-b864-21853efc444e","remote_addr":"[fd00:1122:3344:101::a]:44015","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":86,"response_code":"200"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-ff906436-1220-4941-af9d-a7161cedb434","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.063218347Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.072609692Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-ff906436-1220-4941-af9d-a7161cedb434","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.075111526Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/ff906436-1220-4941-af9d-a7161cedb434","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.07537071Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.075425071Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.075468502Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.075510253Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.079198617Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:56:59.083522873Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.262993373Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee","method":"GET","req_id":"14a41ca0-19cc-4b15-87fc-1b8a23072a95","remote_addr":"[fd00:1122:3344:101::a]:44015","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":57,"response_code":"200"}
{"msg":"path is \"/data/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.266338451Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"dataset is oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.27537529Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"snapshot is oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee@da1b3983-872e-49aa-8179-62bbc67dd637","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.284041372Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee\", \"da1b3983-872e-49aa-8179-62bbc67dd637\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.284083342Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.294753929Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee/snapshots/da1b3983-872e-49aa-8179-62bbc67dd637","method":"GET","req_id":"e322c209-214e-41bd-9b28-088da8cdd715","remote_addr":"[fd00:1122:3344:101::a]:44015","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":28868,"response_code":"200"}
{"msg":"path is \"/data/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.29820314Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"dataset is oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.305914955Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"snapshot is oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee@da1b3983-872e-49aa-8179-62bbc67dd637","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.31531574Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee\", \"da1b3983-872e-49aa-8179-62bbc67dd637\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.315344401Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"ZfsSnapshotInterface"}
{"msg":"requesting running snapshot 94abf463-bb77-4473-9ad5-5bd1b25690ee-da1b3983-872e-49aa-8179-62bbc67dd637 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.323909691Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"applying SMF actions for region 94abf463-bb77-4473-9ad5-5bd1b25690ee running snapshot da1b3983-872e-49aa-8179-62bbc67dd637 (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.324115484Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.324135135Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee/snapshots/da1b3983-872e-49aa-8179-62bbc67dd637/run","method":"POST","req_id":"67d06243-7327-4751-9a83-6ef081da8437","remote_addr":"[fd00:1122:3344:101::a]:44015","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":25966,"response_code":"200"}
{"msg":"creating missing snapshot instance snapshot-94abf463-bb77-4473-9ad5-5bd1b25690ee-da1b3983-872e-49aa-8179-62bbc67dd637","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.326852722Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-94abf463-bb77-4473-9ad5-5bd1b25690ee-da1b3983-872e-49aa-8179-62bbc67dd637","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.329440377Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-94abf463-bb77-4473-9ad5-5bd1b25690ee-da1b3983-872e-49aa-8179-62bbc67dd637","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.339085986Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee/.zfs/snapshot/da1b3983-872e-49aa-8179-62bbc67dd637","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.33929115Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.339344581Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.339390812Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.339429523Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.339478924Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.34271301Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.346766491Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"worker"}
{"msg":"region 94abf463-bb77-4473-9ad5-5bd1b25690ee running snapshot da1b3983-872e-49aa-8179-62bbc67dd637 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.346778811Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"path is \"/data/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee\"","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.666343073Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"dataset is oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.675587155Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"datafile"}
{"msg":"snapshot is oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee@da1b3983-872e-49aa-8179-62bbc67dd637","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.684202875Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"ZfsSnapshotInterface"}
{"msg":"parts is [\"oxp_e43f0729-982f-4960-b10a-7966152ae48a/crucible/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee\", \"da1b3983-872e-49aa-8179-62bbc67dd637\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.684237816Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"component":"ZfsSnapshotInterface"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-11-13T20:57:01.692658764Z","hostname":"oxz_crucible_15ac7357-6094-4eed-90df-96d45539e934","pid":28574,"uri":"/crucible/0/regions/94abf463-bb77-4473-9ad5-5bd1b25690ee/snapshots/da1b3983-872e-49aa-8179-62bbc67dd637/run","method":"POST","req_id":"8ad199f0-516c-4199-8db3-85d09c89429d","remote_addr":"[fd00:1122:3344:101::a]:44015","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26356,"response_code":"200"}