[ Sep 23 23:12:12 Enabled. ]
[ Sep 23 23:12:13 Rereading configuration. ]
[ Sep 23 23:12:14 Rereading configuration. ]
[ Sep 23 23:12:15 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_fd5ca1dd-39f0-4844-af6b-77dcaa746e9d/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_fd5ca1dd-39f0-4844-af6b-77dcaa746e9d/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:15.786368738Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182}
{"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:15.786743852Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:15.786753532Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:15.809309772Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:15.94011181Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.383509226Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:61942"}
{"msg":"region e617757d-65ed-411c-aaab-4220c112c3d8 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.410539608Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.414566364Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"uri":"/crucible/0/regions","method":"POST","req_id":"a2ffdd09-bc03-48e2-9672-7f61d10250f4","remote_addr":"[fd00:1122:3344:101::b]:61942","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":17807,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.414590574Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_fd5ca1dd-39f0-4844-af6b-77dcaa746e9d/crucible/regions/e617757d-65ed-411c-aaab-4220c112c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.425209377Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_fd5ca1dd-39f0-4844-af6b-77dcaa746e9d/crucible/regions/e617757d-65ed-411c-aaab-4220c112c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.425250527Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"e617757d-65ed-411c-aaab-4220c112c3d8\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/e617757d-65ed-411c-aaab-4220c112c3d8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.478047296Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"region":"e617757d-65ed-411c-aaab-4220c112c3d8","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"e617757d-65ed-411c-aaab-4220c112c3d8\" \"--data\" \"/data/regions/e617757d-65ed-411c-aaab-4220c112c3d8\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.482564918Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"region":"e617757d-65ed-411c-aaab-4220c112c3d8","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.588883663Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"region":"e617757d-65ed-411c-aaab-4220c112c3d8","component":"worker"}
{"msg":"region e617757d-65ed-411c-aaab-4220c112c3d8 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.588964994Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.589124296Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-e617757d-65ed-411c-aaab-4220c112c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.591779426Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-e617757d-65ed-411c-aaab-4220c112c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.595846433Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.606316594Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-e617757d-65ed-411c-aaab-4220c112c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.610505262Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/e617757d-65ed-411c-aaab-4220c112c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.618213091Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.620597159Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.620649059Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.62069291Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.625493675Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.632590217Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:44.740761423Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"uri":"/crucible/0/regions","method":"POST","req_id":"d626a8bf-0ed9-4b0f-b6f0-afd2af3aab85","remote_addr":"[fd00:1122:3344:101::b]:61942","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":77,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.475103557Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"uri":"/crucible/0/regions/e617757d-65ed-411c-aaab-4220c112c3d8","method":"GET","req_id":"9c4c52cc-54d7-49d7-b4ba-298de1147537","remote_addr":"[fd00:1122:3344:101::b]:61942","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":67,"response_code":"200"}
{"msg":"path is \"/data/regions/e617757d-65ed-411c-aaab-4220c112c3d8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.586975766Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"datafile"}
{"msg":"dataset is oxp_fd5ca1dd-39f0-4844-af6b-77dcaa746e9d/crucible/regions/e617757d-65ed-411c-aaab-4220c112c3d8","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.596380145Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"datafile"}
{"msg":"region e617757d-65ed-411c-aaab-4220c112c3d8 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.605861894Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.606098247Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.606175628Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"uri":"/crucible/0/regions/e617757d-65ed-411c-aaab-4220c112c3d8","method":"DELETE","req_id":"86c6c046-5173-489f-b40a-ee7019ab3374","remote_addr":"[fd00:1122:3344:101::b]:61942","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19265,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-e617757d-65ed-411c-aaab-4220c112c3d8 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.607188829Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.609772449Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"uri":"/crucible/0/regions/e617757d-65ed-411c-aaab-4220c112c3d8","method":"GET","req_id":"a42e7b3a-9480-4759-b9ee-f120688e1cee","remote_addr":"[fd00:1122:3344:101::b]:61942","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":32,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.611911794Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_fd5ca1dd-39f0-4844-af6b-77dcaa746e9d/crucible/regions/e617757d-65ed-411c-aaab-4220c112c3d8\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.621824188Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"region":"e617757d-65ed-411c-aaab-4220c112c3d8","component":"worker"}
{"msg":"region e617757d-65ed-411c-aaab-4220c112c3d8 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.685836736Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:12:45.715338636Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"uri":"/crucible/0/regions/e617757d-65ed-411c-aaab-4220c112c3d8","method":"GET","req_id":"b6c78ab3-a7df-4e39-b5f1-2250773b7884","remote_addr":"[fd00:1122:3344:101::b]:61942","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":671,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.556476309Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:45565"}
{"msg":"region 192f05bc-462d-4c7c-9cca-1b1bfc2111b1 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.55660427Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"datafile"}
{"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.556794313Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.556810093Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"uri":"/crucible/0/regions","method":"POST","req_id":"0886f3f7-b532-4ec3-8add-1ca00207bcd2","remote_addr":"[fd00:1122:3344:101::a]:45565","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":230,"response_code":"200"}
{"msg":"zfs set reservation of 2684354560 for oxp_fd5ca1dd-39f0-4844-af6b-77dcaa746e9d/crucible/regions/192f05bc-462d-4c7c-9cca-1b1bfc2111b1","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.565485003Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"zfs set quota of 6442450944 for oxp_fd5ca1dd-39f0-4844-af6b-77dcaa746e9d/crucible/regions/192f05bc-462d-4c7c-9cca-1b1bfc2111b1","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.565522683Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"192f05bc-462d-4c7c-9cca-1b1bfc2111b1\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/192f05bc-462d-4c7c-9cca-1b1bfc2111b1\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.614414587Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"region":"192f05bc-462d-4c7c-9cca-1b1bfc2111b1","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"192f05bc-462d-4c7c-9cca-1b1bfc2111b1\" \"--data\" \"/data/regions/192f05bc-462d-4c7c-9cca-1b1bfc2111b1\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.614463157Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"region":"192f05bc-462d-4c7c-9cca-1b1bfc2111b1","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.700349577Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"region":"192f05bc-462d-4c7c-9cca-1b1bfc2111b1","component":"worker"}
{"msg":"region 192f05bc-462d-4c7c-9cca-1b1bfc2111b1 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.700393398Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.700513709Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-192f05bc-462d-4c7c-9cca-1b1bfc2111b1","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.702111537Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-192f05bc-462d-4c7c-9cca-1b1bfc2111b1","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.705260054Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.714334138Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-192f05bc-462d-4c7c-9cca-1b1bfc2111b1","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.721645642Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/192f05bc-462d-4c7c-9cca-1b1bfc2111b1","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.721944096Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.722007667Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.722052377Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.722094548Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.726431878Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.732139174Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-09-23T23:13:40.856673919Z","hostname":"oxz_crucible_3a08792a-4a03-479c-8556-d27ce039902e","pid":22182,"uri":"/crucible/0/regions","method":"POST","req_id":"0d1488f3-28f2-4dcf-ba2b-105d1070298e","remote_addr":"[fd00:1122:3344:101::a]:45565","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":64,"response_code":"200"}