{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.413137714Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:61717"} {"msg":"region 3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.443794727Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.448037089Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"uri":"/crucible/0/regions","method":"POST","req_id":"11da36e0-a524-4148-8706-3b7e503ef5d7","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":12009,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.449221751Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_0ee78aa3-d1dc-4aba-bf73-48e592f89cd6/crucible/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.459581695Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_0ee78aa3-d1dc-4aba-bf73-48e592f89cd6/crucible/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.459612906Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.520965422Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"region":"3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a\" \"--data\" \"/data/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.522940095Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"region":"3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.610840325Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"uri":"/crucible/0/regions","method":"POST","req_id":"f379d85f-3167-429a-b3c0-0fe9c3bfb452","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":87,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.670162788Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"region":"3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","component":"worker"} {"msg":"region 3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.670210549Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.670307912Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.672539631Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.677183884Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.694629396Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.698218392Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.703780639Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.703845271Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::12","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.703888532Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.703934463Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.70760079Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:03.711876074Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:04.273981224Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"uri":"/crucible/0/regions","method":"POST","req_id":"c59edac3-9a3b-4550-837a-abc1c56ebe32","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:04.455531428Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:52419"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:04.455717122Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"uri":"/crucible/0/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","method":"GET","req_id":"f3d22082-1b3d-4bcc-b425-eb73a7f85f84","remote_addr":"[fd00:1122:3344:101::a]:52419","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:04.482914113Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:33941"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:04.483012196Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"uri":"/crucible/0/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","method":"GET","req_id":"65c1f8b6-748d-4fce-a038-dae818f4714a","remote_addr":"[fd00:1122:3344:101::b]:33941","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:04.801105679Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:63607"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:04.801201691Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"uri":"/crucible/0/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","method":"GET","req_id":"38dd0669-1105-4286-9f64-07d182ff96e6","remote_addr":"[fd00:1122:3344:101::c]:63607","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":45,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.495467566Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"uri":"/crucible/0/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","method":"GET","req_id":"323b8303-882a-4a29-b182-a9bb19aac751","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":48,"response_code":"200"} {"msg":"path is \"/data/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.498141467Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"datafile"} {"msg":"dataset is oxp_0ee78aa3-d1dc-4aba-bf73-48e592f89cd6/crucible/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.509755964Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"datafile"} {"msg":"region 3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.520195801Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.520362196Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.520437348Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"uri":"/crucible/0/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","method":"DELETE","req_id":"c42b48ab-147f-4cf4-bd03-0001d887b738","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":22326,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.521439594Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.5227966Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"uri":"/crucible/0/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","method":"GET","req_id":"047dcf9e-1110-472f-8eb7-7d2910acb34f","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":38,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.525284346Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_0ee78aa3-d1dc-4aba-bf73-48e592f89cd6/crucible/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.536957416Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"region":"3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","component":"worker"} {"msg":"region 3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.60012234Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-21T22:16:05.629793477Z","hostname":"oxz_crucible_f1d12ea4-18ab-4104-ba92-be8abd456ce0","pid":21230,"uri":"/crucible/0/regions/3cbe252f-9f0d-47c9-82f5-bc8db9fb1b4a","method":"GET","req_id":"4cc55abe-9015-41fb-b538-afb430a4d1eb","remote_addr":"[fd00:1122:3344:101::c]:61717","local_addr":"[fd00:1122:3344:101::12]:32345","component":"dropshot","latency_us":79,"response_code":"200"}