[ Jun 4 18:27:57 Enabled. ] [ Jun 4 18:27:57 Rereading configuration. ] [ Jun 4 18:27:59 Rereading configuration. ] [ Jun 4 18:28:03 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_cbe647df-2d83-4b35-bc78-aff3bdabdce3/crucible -l [fd00:1122:3344:101::16]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_cbe647df-2d83-4b35-bc78-aff3bdabdce3/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:03.360601599Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569} {"msg":"listen IP: [fd00:1122:3344:101::16]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:03.360979634Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:03.360990525Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:03.380935323Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:03.467650336Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:34.687776998Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:39879"} {"msg":"region aa4feb9b-14a4-4294-896d-5b08e47f31a3 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:34.707621104Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:34.711815131Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"uri":"/crucible/0/regions","method":"POST","req_id":"eaefb38b-44d0-4068-9589-1b4e87209e9e","remote_addr":"[fd00:1122:3344:101::a]:39879","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":16341,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:34.713991236Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_cbe647df-2d83-4b35-bc78-aff3bdabdce3/crucible/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:34.724178988Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_cbe647df-2d83-4b35-bc78-aff3bdabdce3/crucible/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:34.724243129Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"aa4feb9b-14a4-4294-896d-5b08e47f31a3\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:34.807762671Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"region":"aa4feb9b-14a4-4294-896d-5b08e47f31a3","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"aa4feb9b-14a4-4294-896d-5b08e47f31a3\" \"--data\" \"/data/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:34.809799874Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"region":"aa4feb9b-14a4-4294-896d-5b08e47f31a3","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:34.919512194Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"uri":"/crucible/0/regions","method":"POST","req_id":"e1ddeb89-2d12-4838-af82-7109a2dbe54a","remote_addr":"[fd00:1122:3344:101::a]:39879","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":71,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.068821485Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"region":"aa4feb9b-14a4-4294-896d-5b08e47f31a3","component":"worker"} {"msg":"region aa4feb9b-14a4-4294-896d-5b08e47f31a3 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.068890266Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.069078719Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-aa4feb9b-14a4-4294-896d-5b08e47f31a3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.069469285Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-aa4feb9b-14a4-4294-896d-5b08e47f31a3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.077078136Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.095859706Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-aa4feb9b-14a4-4294-896d-5b08e47f31a3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.099479573Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.100289517Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.100372858Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.100422869Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.100469069Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.105205225Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.110685502Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.415085337Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"uri":"/crucible/0/regions","method":"POST","req_id":"81a5ecae-bf03-4927-8bd7-979ef8d2da65","remote_addr":"[fd00:1122:3344:101::a]:39879","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":42,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.885407148Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"uri":"/crucible/0/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3","method":"GET","req_id":"7762b003-34b6-441b-9ed1-a19e408693b8","remote_addr":"[fd00:1122:3344:101::a]:39879","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":77,"response_code":"200"} {"msg":"path is \"/data/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.889114887Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"datafile"} {"msg":"dataset is oxp_cbe647df-2d83-4b35-bc78-aff3bdabdce3/crucible/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.900169683Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"datafile"} {"msg":"region aa4feb9b-14a4-4294-896d-5b08e47f31a3 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.910697481Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.910880784Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.910932615Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"uri":"/crucible/0/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3","method":"DELETE","req_id":"68aab22f-750c-42ca-a7d8-fe09ef0ee297","remote_addr":"[fd00:1122:3344:101::a]:39879","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":21837,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-aa4feb9b-14a4-4294-896d-5b08e47f31a3 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.91191054Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.913721789Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"uri":"/crucible/0/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3","method":"GET","req_id":"418e3d60-9c84-4c0b-b650-b488785d2197","remote_addr":"[fd00:1122:3344:101::a]:39879","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":22,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.922204405Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_cbe647df-2d83-4b35-bc78-aff3bdabdce3/crucible/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:35.933331642Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"region":"aa4feb9b-14a4-4294-896d-5b08e47f31a3","component":"worker"} {"msg":"region aa4feb9b-14a4-4294-896d-5b08e47f31a3 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:36.013826116Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:36.031943515Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"uri":"/crucible/0/regions/aa4feb9b-14a4-4294-896d-5b08e47f31a3","method":"GET","req_id":"144a9235-7355-4911-abdc-490709424f4b","remote_addr":"[fd00:1122:3344:101::a]:39879","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":55,"response_code":"200"} {"msg":"region 8b9e7149-f54b-48ef-9d41-96e41fa95174 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.289477687Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.289611939Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.289626569Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"uri":"/crucible/0/regions","method":"POST","req_id":"ddc33ca3-2254-47db-b87b-60d97bc5d184","remote_addr":"[fd00:1122:3344:101::a]:39879","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":195,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_cbe647df-2d83-4b35-bc78-aff3bdabdce3/crucible/regions/8b9e7149-f54b-48ef-9d41-96e41fa95174","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.299331204Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_cbe647df-2d83-4b35-bc78-aff3bdabdce3/crucible/regions/8b9e7149-f54b-48ef-9d41-96e41fa95174","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.299363075Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"8b9e7149-f54b-48ef-9d41-96e41fa95174\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/8b9e7149-f54b-48ef-9d41-96e41fa95174\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.360356968Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"region":"8b9e7149-f54b-48ef-9d41-96e41fa95174","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"8b9e7149-f54b-48ef-9d41-96e41fa95174\" \"--data\" \"/data/regions/8b9e7149-f54b-48ef-9d41-96e41fa95174\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.360407828Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"region":"8b9e7149-f54b-48ef-9d41-96e41fa95174","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.460633917Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"region":"8b9e7149-f54b-48ef-9d41-96e41fa95174","component":"worker"} {"msg":"region 8b9e7149-f54b-48ef-9d41-96e41fa95174 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.460726768Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.460880761Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-8b9e7149-f54b-48ef-9d41-96e41fa95174","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.462507647Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-8b9e7149-f54b-48ef-9d41-96e41fa95174","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.466433139Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.475940771Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-8b9e7149-f54b-48ef-9d41-96e41fa95174","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.481400978Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/8b9e7149-f54b-48ef-9d41-96e41fa95174","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.481610711Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.481666622Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::16","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.481705513Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.481750264Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.48839615Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.494997635Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-06-04T18:28:38.511111062Z","hostname":"oxz_crucible_9032488f-4a52-4ca8-85ce-a4bbab527de0","pid":22569,"uri":"/crucible/0/regions","method":"POST","req_id":"01062d8c-5eba-40de-a918-052f94b8be0f","remote_addr":"[fd00:1122:3344:101::a]:39879","local_addr":"[fd00:1122:3344:101::16]:32345","component":"dropshot","latency_us":54,"response_code":"200"}