[ Feb 22 01:10:44 Enabled. ]
[ Feb 22 01:10:44 Rereading configuration. ]
[ Feb 22 01:10:46 Rereading configuration. ]
[ Feb 22 01:10:49 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:10:49.24790571Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:10:49.249812776Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:10:49.249823966Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:10:49.383437786Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.89999406Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:64486"}
{"msg":"region 49f7eee6-d4c3-478b-8555-12fa4c5402f4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.954467062Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.954684898Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"uri":"/crucible/0/regions","method":"POST","req_id":"6daa6a88-868b-4939-905e-d7ad5736f845","remote_addr":"[fd00:1122:3344:101::b]:64486","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":5328,"response_code":"200"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.955291916Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"zfs set reservation of 1342177280 for oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/49f7eee6-d4c3-478b-8555-12fa4c5402f4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.964519154Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/49f7eee6-d4c3-478b-8555-12fa4c5402f4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:42.964555055Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"49f7eee6-d4c3-478b-8555-12fa4c5402f4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None } at \"/data/regions/49f7eee6-d4c3-478b-8555-12fa4c5402f4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.014613078Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"region":"49f7eee6-d4c3-478b-8555-12fa4c5402f4","component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.105448186Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"uri":"/crucible/0/regions","method":"POST","req_id":"f6a91edb-dc2a-426e-80cc-33aca850e301","remote_addr":"[fd00:1122:3344:101::b]:64486","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":83,"response_code":"200"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.124730386Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"region":"49f7eee6-d4c3-478b-8555-12fa4c5402f4","component":"worker"}
{"msg":"region 49f7eee6-d4c3-478b-8555-12fa4c5402f4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.124759437Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.12487984Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-49f7eee6-d4c3-478b-8555-12fa4c5402f4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.125304392Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-49f7eee6-d4c3-478b-8555-12fa4c5402f4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.129046401Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.139071872Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-49f7eee6-d4c3-478b-8555-12fa4c5402f4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.143495801Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/49f7eee6-d4c3-478b-8555-12fa4c5402f4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.144213572Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.144380786Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.144423448Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.144463949Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.14864316Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.152898324Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.723362529Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"uri":"/crucible/0/regions","method":"POST","req_id":"675224a9-d24d-4647-a0ba-c5364f58df35","remote_addr":"[fd00:1122:3344:101::b]:64486","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":70,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.942166671Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::b]:61622"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.942418659Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"uri":"/crucible/0/regions/49f7eee6-d4c3-478b-8555-12fa4c5402f4","method":"GET","req_id":"db369afa-7f70-46ed-944c-9124b16b3806","remote_addr":"[fd00:1122:3344:101::b]:61622","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":159,"response_code":"200"}
{"msg":"path is \"/data/regions/49f7eee6-d4c3-478b-8555-12fa4c5402f4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.942637835Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"datafile"}
{"msg":"dataset is oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/49f7eee6-d4c3-478b-8555-12fa4c5402f4","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.951437861Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"datafile"}
{"msg":"region 49f7eee6-d4c3-478b-8555-12fa4c5402f4 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.960103222Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.960257287Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.960324529Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"uri":"/crucible/0/regions/49f7eee6-d4c3-478b-8555-12fa4c5402f4","method":"DELETE","req_id":"83b94e55-1e36-4c23-83c5-355885f97c25","remote_addr":"[fd00:1122:3344:101::b]:61622","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":17710,"response_code":"204"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.960512414Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"uri":"/crucible/0/regions/49f7eee6-d4c3-478b-8555-12fa4c5402f4","method":"GET","req_id":"ae8e9ac7-6f10-4ca1-802c-33b52c14dc73","remote_addr":"[fd00:1122:3344:101::b]:61622","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":22,"response_code":"200"}
{"msg":"disabling downstairs instance: downstairs-49f7eee6-d4c3-478b-8555-12fa4c5402f4 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.961549684Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.965675384Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03/crucible/regions/49f7eee6-d4c3-478b-8555-12fa4c5402f4\"","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:43.975700855Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"region":"49f7eee6-d4c3-478b-8555-12fa4c5402f4","component":"worker"}
{"msg":"region 49f7eee6-d4c3-478b-8555-12fa4c5402f4 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.026774548Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-02-22T01:11:44.091375544Z","hostname":"oxz_crucible_6ae246f7-528e-47a2-b40e-5230de2e2247","pid":23501,"uri":"/crucible/0/regions/49f7eee6-d4c3-478b-8555-12fa4c5402f4","method":"GET","req_id":"dd7ab18a-ef7d-4897-9d2f-b0962f83f83b","remote_addr":"[fd00:1122:3344:101::b]:61622","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":595,"response_code":"200"}