[ Aug 3 04:36:43 Enabled. ] [ Aug 3 04:36:43 Rereading configuration. ] [ Aug 3 04:36:47 Rereading configuration. ] [ Aug 3 04:36:53 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_2d57b90c-5604-40fb-87e1-361dc611bf9e/crucible -l [fd00:1122:3344:101::18]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_2d57b90c-5604-40fb-87e1-361dc611bf9e/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:36:53.855802095Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339} {"msg":"listen IP: [fd00:1122:3344:101::18]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:36:53.857070299Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:36:53.857082939Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:36:53.876483498Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:36:54.021695929Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.175222983Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:47301"} {"msg":"region 7fbaccf9-5b78-4ec6-a387-b001c7c26dfa state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.20375417Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"datafile"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.211001948Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.215799679Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"uri":"/crucible/0/regions","method":"POST","req_id":"bee2e7eb-ddc9-4888-9a81-e1e9d88ee603","remote_addr":"[fd00:1122:3344:101::c]:47301","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":20325,"response_code":"200"} {"msg":"zfs set reservation of 1342177280 for oxp_2d57b90c-5604-40fb-87e1-361dc611bf9e/crucible/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.224090098Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_2d57b90c-5604-40fb-87e1-361dc611bf9e/crucible/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.224114779Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"7fbaccf9-5b78-4ec6-a387-b001c7c26dfa\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.286312047Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"region":"7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"7fbaccf9-5b78-4ec6-a387-b001c7c26dfa\" \"--data\" \"/data/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.288689333Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"region":"7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.504800268Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"region":"7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","component":"worker"} {"msg":"region 7fbaccf9-5b78-4ec6-a387-b001c7c26dfa state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.504858668Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.504936169Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.505338523Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.512024805Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.531107491Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.535023762Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.542874697Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.545232902Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::18","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.545285543Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.545326013Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.550102425Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.554828035Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:49.582756616Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"uri":"/crucible/0/regions","method":"POST","req_id":"4aa269b6-63c3-4484-b858-8b3cfa1e79e0","remote_addr":"[fd00:1122:3344:101::c]:47301","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":80,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.262453616Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"uri":"/crucible/0/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","method":"GET","req_id":"57a4a866-6d28-4fcd-8703-d55802f302d5","remote_addr":"[fd00:1122:3344:101::c]:47301","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":76,"response_code":"200"} {"msg":"path is \"/data/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.265344147Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"datafile"} {"msg":"dataset is oxp_2d57b90c-5604-40fb-87e1-361dc611bf9e/crucible/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.274787599Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"datafile"} {"msg":"region 7fbaccf9-5b78-4ec6-a387-b001c7c26dfa state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.284274521Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.284433912Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.284494073Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"uri":"/crucible/0/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","method":"DELETE","req_id":"5fe5dc53-c354-47f4-8589-4167830cc150","remote_addr":"[fd00:1122:3344:101::c]:47301","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":19175,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-7fbaccf9-5b78-4ec6-a387-b001c7c26dfa (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.285466844Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.287155252Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"uri":"/crucible/0/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","method":"GET","req_id":"2855f9e5-4332-4baa-8504-23c59decd2b4","remote_addr":"[fd00:1122:3344:101::c]:47301","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":25,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.290375686Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_2d57b90c-5604-40fb-87e1-361dc611bf9e/crucible/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa\"","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.299935439Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"region":"7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","component":"worker"} {"msg":"region 7fbaccf9-5b78-4ec6-a387-b001c7c26dfa state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.350765386Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-08-03T04:37:50.408879961Z","hostname":"oxz_crucible_a3a39c6e-e40a-48d9-9e57-299f9d472923","pid":21339,"uri":"/crucible/0/regions/7fbaccf9-5b78-4ec6-a387-b001c7c26dfa","method":"GET","req_id":"da1b95f9-d567-40c1-b9a8-35dd51dcab35","remote_addr":"[fd00:1122:3344:101::c]:47301","local_addr":"[fd00:1122:3344:101::18]:32345","component":"dropshot","latency_us":56,"response_code":"200"}