[ Oct 15 18:33:48 Enabled. ]
[ Oct 15 18:33:48 Rereading configuration. ]
[ Oct 15 18:33:49 Rereading configuration. ]
[ Oct 15 18:33:50 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_16c9b9f3-ec41-4058-9aa6-27bbfb9792d3/crucible -l [fd00:1122:3344:101::14]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_16c9b9f3-ec41-4058-9aa6-27bbfb9792d3/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:50.655330085Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107}
{"msg":"listen IP: [fd00:1122:3344:101::14]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:50.657220857Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:50.657230797Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:50.678487569Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:33:50.784034304Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.093172257Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:61323"}
{"msg":"region 524c49ec-f48e-4d3a-a4d1-850696ee5961 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.128396309Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.138678326Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"uri":"/crucible/0/regions","method":"POST","req_id":"8f8c532f-9666-444a-93a7-67b974a8d51f","remote_addr":"[fd00:1122:3344:101::c]:61323","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":18959,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.138704487Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_16c9b9f3-ec41-4058-9aa6-27bbfb9792d3/crucible/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.148283906Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_16c9b9f3-ec41-4058-9aa6-27bbfb9792d3/crucible/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.148312746Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"524c49ec-f48e-4d3a-a4d1-850696ee5961\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.250799235Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"region":"524c49ec-f48e-4d3a-a4d1-850696ee5961","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"524c49ec-f48e-4d3a-a4d1-850696ee5961\" \"--data\" \"/data/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.254615589Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"region":"524c49ec-f48e-4d3a-a4d1-850696ee5961","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.30291702Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"uri":"/crucible/0/regions","method":"POST","req_id":"d59a1617-43df-43af-868b-fc982c792adf","remote_addr":"[fd00:1122:3344:101::c]:61323","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":76,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.548783994Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"region":"524c49ec-f48e-4d3a-a4d1-850696ee5961","component":"worker"}
{"msg":"region 524c49ec-f48e-4d3a-a4d1-850696ee5961 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.548825615Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.548936136Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-524c49ec-f48e-4d3a-a4d1-850696ee5961","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.551621997Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-524c49ec-f48e-4d3a-a4d1-850696ee5961","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.557757496Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.577038916Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-524c49ec-f48e-4d3a-a4d1-850696ee5961","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.583328158Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.592442202Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.592506583Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::14","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.592546564Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.592585014Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.597072055Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.604846214Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:53.666050352Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"uri":"/crucible/0/regions","method":"POST","req_id":"8451d1cf-ab9d-48f7-8bbe-34ba8656a76a","remote_addr":"[fd00:1122:3344:101::c]:61323","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":74,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.431627185Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"uri":"/crucible/0/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961","method":"GET","req_id":"dfd05720-bd13-4298-aa89-a7f3034b4682","remote_addr":"[fd00:1122:3344:101::c]:61323","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":81,"response_code":"200"}
{"msg":"path is \"/data/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.435380147Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"datafile"}
{"msg":"dataset is oxp_16c9b9f3-ec41-4058-9aa6-27bbfb9792d3/crucible/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.446599366Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"datafile"}
{"msg":"region 524c49ec-f48e-4d3a-a4d1-850696ee5961 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.457119566Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.457272407Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.457370288Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"uri":"/crucible/0/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961","method":"DELETE","req_id":"49ddf8c0-070b-4208-896f-1adf68cce255","remote_addr":"[fd00:1122:3344:101::c]:61323","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":22023,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-524c49ec-f48e-4d3a-a4d1-850696ee5961 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.45839382Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.461689178Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"uri":"/crucible/0/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961","method":"GET","req_id":"f70aa75f-b93c-4825-95b6-2b15a286af96","remote_addr":"[fd00:1122:3344:101::c]:61323","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":39,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.465237848Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_16c9b9f3-ec41-4058-9aa6-27bbfb9792d3/crucible/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961\"","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.476035591Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"region":"524c49ec-f48e-4d3a-a4d1-850696ee5961","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.517540995Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"uri":"/crucible/0/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961","method":"GET","req_id":"83b741ed-940c-4a82-9f6d-f0cc513c03ee","remote_addr":"[fd00:1122:3344:101::c]:61323","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":84,"response_code":"200"}
{"msg":"region 524c49ec-f48e-4d3a-a4d1-850696ee5961 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.541723101Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-10-15T18:34:54.695431484Z","hostname":"oxz_crucible_a372449f-a2b0-4d65-8db5-c7348064f145","pid":20107,"uri":"/crucible/0/regions/524c49ec-f48e-4d3a-a4d1-850696ee5961","method":"GET","req_id":"4b15512d-31af-49c1-a6e8-0a99c2e50101","remote_addr":"[fd00:1122:3344:101::c]:61323","local_addr":"[fd00:1122:3344:101::14]:32345","component":"dropshot","latency_us":44,"response_code":"200"}