[ Dec 20 22:53:46 Enabled. ]
[ Dec 20 22:53:46 Rereading configuration. ]
[ Dec 20 22:53:48 Rereading configuration. ]
[ Dec 20 22:53:52 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_38fd6a55-6517-4c3e-af4d-ade80ff38aae/crucible -l [fd00:1122:3344:101::1a]:32345 -P 19000 -p downstairs -s snapshot"). ]
note: configured to log to "/dev/stdout"
{"msg":"dataset: \"oxp_38fd6a55-6517-4c3e-af4d-ade80ff38aae/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:53:52.190011864Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200}
{"msg":"listen IP: [fd00:1122:3344:101::1a]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:53:52.195796742Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200}
{"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:53:52.195811953Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200}
{"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:53:52.212130285Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"datafile"}
{"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:53:52.490039101Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot"}
{"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.161020215Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:44649"}
{"msg":"region 615bb997-d143-49d6-a1db-7349903f8c3e state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.208931391Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"datafile"}
{"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.213751703Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.218676388Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"uri":"/crucible/0/regions","method":"POST","req_id":"ae704bc4-9759-4d9b-a4ca-34cf94485ab2","remote_addr":"[fd00:1122:3344:101::a]:44649","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":16026,"response_code":"200"}
{"msg":"zfs set reservation of 1342177280 for oxp_38fd6a55-6517-4c3e-af4d-ade80ff38aae/crucible/regions/615bb997-d143-49d6-a1db-7349903f8c3e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.224919979Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"zfs set quota of 3221225472 for oxp_38fd6a55-6517-4c3e-af4d-ade80ff38aae/crucible/regions/615bb997-d143-49d6-a1db-7349903f8c3e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.22495489Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"creating region Region { id: RegionId(\"615bb997-d143-49d6-a1db-7349903f8c3e\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/615bb997-d143-49d6-a1db-7349903f8c3e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.278945803Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"region":"615bb997-d143-49d6-a1db-7349903f8c3e","component":"worker"}
{"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"615bb997-d143-49d6-a1db-7349903f8c3e\" \"--data\" \"/data/regions/615bb997-d143-49d6-a1db-7349903f8c3e\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.280983329Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"region":"615bb997-d143-49d6-a1db-7349903f8c3e","component":"worker"}
{"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.385334274Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"region":"615bb997-d143-49d6-a1db-7349903f8c3e","component":"worker"}
{"msg":"region 615bb997-d143-49d6-a1db-7349903f8c3e state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.385387005Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"datafile"}
{"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.385475108Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"creating missing downstairs instance downstairs-615bb997-d143-49d6-a1db-7349903f8c3e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.385862318Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-615bb997-d143-49d6-a1db-7349903f8c3e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.394735472Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.408088959Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-615bb997-d143-49d6-a1db-7349903f8c3e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.411123242Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/615bb997-d143-49d6-a1db-7349903f8c3e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.411870023Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.411936045Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::1a","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.411983526Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.412024557Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.416025367Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.420212872Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.582865998Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"uri":"/crucible/0/regions","method":"POST","req_id":"e53064b8-7433-4b62-90c9-599c327fb330","remote_addr":"[fd00:1122:3344:101::a]:44649","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":73,"response_code":"200"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.936490898Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"uri":"/crucible/0/regions/615bb997-d143-49d6-a1db-7349903f8c3e","method":"GET","req_id":"f3eb6c9e-237e-4b4b-8f3e-a3d589d71293","remote_addr":"[fd00:1122:3344:101::a]:44649","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":78,"response_code":"200"}
{"msg":"path is \"/data/regions/615bb997-d143-49d6-a1db-7349903f8c3e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.939651475Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"datafile"}
{"msg":"dataset is oxp_38fd6a55-6517-4c3e-af4d-ade80ff38aae/crucible/regions/615bb997-d143-49d6-a1db-7349903f8c3e","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.950674137Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"datafile"}
{"msg":"region 615bb997-d143-49d6-a1db-7349903f8c3e state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.960081516Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"datafile"}
{"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.96022874Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.960286831Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"uri":"/crucible/0/regions/615bb997-d143-49d6-a1db-7349903f8c3e","method":"DELETE","req_id":"e421d977-9d93-438d-a49b-a21b10efa4d5","remote_addr":"[fd00:1122:3344:101::a]:44649","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":20673,"response_code":"204"}
{"msg":"disabling downstairs instance: downstairs-615bb997-d143-49d6-a1db-7349903f8c3e (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.961308439Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.963377176Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"uri":"/crucible/0/regions/615bb997-d143-49d6-a1db-7349903f8c3e","method":"GET","req_id":"7cda38c8-5cf7-48f0-ae79-9e83bbd1aa67","remote_addr":"[fd00:1122:3344:101::a]:44649","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":37,"response_code":"200"}
{"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.965621068Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"worker"}
{"msg":"deleting zfs dataset \"oxp_38fd6a55-6517-4c3e-af4d-ade80ff38aae/crucible/regions/615bb997-d143-49d6-a1db-7349903f8c3e\"","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:47.976343342Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"region":"615bb997-d143-49d6-a1db-7349903f8c3e","component":"worker"}
{"msg":"region 615bb997-d143-49d6-a1db-7349903f8c3e state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:48.03017683Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"component":"datafile"}
{"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-12-20T22:54:48.061426788Z","hostname":"oxz_crucible_de2736f0-9cb4-49a5-9541-e86eb4dd1a03","pid":22200,"uri":"/crucible/0/regions/615bb997-d143-49d6-a1db-7349903f8c3e","method":"GET","req_id":"057d4036-7f90-4b68-9726-7b70519c58a6","remote_addr":"[fd00:1122:3344:101::a]:44649","local_addr":"[fd00:1122:3344:101::1a]:32345","component":"dropshot","latency_us":65,"response_code":"200"}