[ Mar 13 18:26:30 Enabled. ] [ Mar 13 18:26:30 Rereading configuration. ] [ Mar 13 18:26:33 Rereading configuration. ] [ Mar 13 18:26:38 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_43545d45-2534-4494-aa0d-820d7c438224/crucible -l [fd00:1122:3344:101::19]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_43545d45-2534-4494-aa0d-820d7c438224/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:26:39.027895143Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911} {"msg":"listen IP: [fd00:1122:3344:101::19]:32345","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:26:39.031392494Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:26:39.031400374Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:26:39.038159848Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:26:39.398778078Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:20.727287633Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::c]:45559"} {"msg":"region 9b874fca-fc75-40a8-aed3-1145c22446d4 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:20.773961157Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:20.776479846Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"uri":"/crucible/0/regions","method":"POST","req_id":"b655baad-f897-4083-b344-eccaa26127d2","remote_addr":"[fd00:1122:3344:101::c]:45559","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":18715,"response_code":"200"} {"msg":"Region size:1073741824 reservation:1342177280 quota:3221225472","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:20.778959385Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"zfs set reservation of 1342177280 for oxp_43545d45-2534-4494-aa0d-820d7c438224/crucible/regions/9b874fca-fc75-40a8-aed3-1145c22446d4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:20.790735927Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"zfs set quota of 3221225472 for oxp_43545d45-2534-4494-aa0d-820d7c438224/crucible/regions/9b874fca-fc75-40a8-aed3-1145c22446d4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:20.790765227Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"9b874fca-fc75-40a8-aed3-1145c22446d4\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 16, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/9b874fca-fc75-40a8-aed3-1145c22446d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:20.842909165Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"region":"9b874fca-fc75-40a8-aed3-1145c22446d4","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"9b874fca-fc75-40a8-aed3-1145c22446d4\" \"--data\" \"/data/regions/9b874fca-fc75-40a8-aed3-1145c22446d4\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"16\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:20.845146075Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"region":"9b874fca-fc75-40a8-aed3-1145c22446d4","component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:20.922605491Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"uri":"/crucible/0/regions","method":"POST","req_id":"49153f81-489c-415a-b535-1708c619851f","remote_addr":"[fd00:1122:3344:101::c]:45559","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":80,"response_code":"200"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.133035888Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"region":"9b874fca-fc75-40a8-aed3-1145c22446d4","component":"worker"} {"msg":"region 9b874fca-fc75-40a8-aed3-1145c22446d4 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.133076799Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.133165632Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-9b874fca-fc75-40a8-aed3-1145c22446d4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.133606675Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-9b874fca-fc75-40a8-aed3-1145c22446d4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.138976165Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.152134481Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-9b874fca-fc75-40a8-aed3-1145c22446d4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.155871839Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/9b874fca-fc75-40a8-aed3-1145c22446d4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.156580321Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.156644243Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.156688904Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.156732556Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.160903028Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.17490904Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.322489391Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"uri":"/crucible/0/regions","method":"POST","req_id":"ee8540e3-99c1-45a1-8fe7-b7e478566ca3","remote_addr":"[fd00:1122:3344:101::c]:45559","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":56,"response_code":"200"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.970117896Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"uri":"/crucible/0/regions/9b874fca-fc75-40a8-aed3-1145c22446d4","method":"GET","req_id":"080c083a-3153-4afe-a6ed-a8627eec199c","remote_addr":"[fd00:1122:3344:101::c]:45559","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":2470,"response_code":"200"} {"msg":"path is \"/data/regions/9b874fca-fc75-40a8-aed3-1145c22446d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.973212533Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"datafile"} {"msg":"dataset is oxp_43545d45-2534-4494-aa0d-820d7c438224/crucible/regions/9b874fca-fc75-40a8-aed3-1145c22446d4","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.983519569Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"datafile"} {"msg":"region 9b874fca-fc75-40a8-aed3-1145c22446d4 state: Created -> Tombstoned","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.993022599Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"datafile"} {"msg":"applying SMF actions before removal...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.993184554Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.993237246Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"uri":"/crucible/0/regions/9b874fca-fc75-40a8-aed3-1145c22446d4","method":"DELETE","req_id":"f7e1b148-2f97-44b0-8817-92c7bcaca983","remote_addr":"[fd00:1122:3344:101::c]:45559","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":20069,"response_code":"204"} {"msg":"disabling downstairs instance: downstairs-9b874fca-fc75-40a8-aed3-1145c22446d4 (instance states: (Some(Online), None))","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.994341011Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:21.996710596Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"uri":"/crucible/0/regions/9b874fca-fc75-40a8-aed3-1145c22446d4","method":"GET","req_id":"28818506-004c-44ad-a408-6b70b6e59edd","remote_addr":"[fd00:1122:3344:101::c]:45559","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":26,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:22.003476749Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"deleting zfs dataset \"oxp_43545d45-2534-4494-aa0d-820d7c438224/crucible/regions/9b874fca-fc75-40a8-aed3-1145c22446d4\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:22.016089468Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"region":"9b874fca-fc75-40a8-aed3-1145c22446d4","component":"worker"} {"msg":"region 9b874fca-fc75-40a8-aed3-1145c22446d4 state: Tombstoned -> Destroyed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:22.080220393Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:22.100774803Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"uri":"/crucible/0/regions/9b874fca-fc75-40a8-aed3-1145c22446d4","method":"GET","req_id":"dc210dc6-bc61-400a-ab12-5e110a488382","remote_addr":"[fd00:1122:3344:101::c]:45559","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":76,"response_code":"200"} {"msg":"region 506a02c6-4fb9-4ffa-b363-d1398f5f82cc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.041356844Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"datafile"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.041516659Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.04153768Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"uri":"/crucible/0/regions","method":"POST","req_id":"0d8475bb-d06b-4526-b0fb-e803ef029f44","remote_addr":"[fd00:1122:3344:101::c]:45559","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":236,"response_code":"200"} {"msg":"zfs set reservation of 2684354560 for oxp_43545d45-2534-4494-aa0d-820d7c438224/crucible/regions/506a02c6-4fb9-4ffa-b363-d1398f5f82cc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.050433311Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_43545d45-2534-4494-aa0d-820d7c438224/crucible/regions/506a02c6-4fb9-4ffa-b363-d1398f5f82cc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.050463292Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"506a02c6-4fb9-4ffa-b363-d1398f5f82cc\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/506a02c6-4fb9-4ffa-b363-d1398f5f82cc\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.097771906Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"region":"506a02c6-4fb9-4ffa-b363-d1398f5f82cc","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"506a02c6-4fb9-4ffa-b363-d1398f5f82cc\" \"--data\" \"/data/regions/506a02c6-4fb9-4ffa-b363-d1398f5f82cc\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.099642995Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"region":"506a02c6-4fb9-4ffa-b363-d1398f5f82cc","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.171855076Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"region":"506a02c6-4fb9-4ffa-b363-d1398f5f82cc","component":"worker"} {"msg":"region 506a02c6-4fb9-4ffa-b363-d1398f5f82cc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.171893987Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.171999831Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-506a02c6-4fb9-4ffa-b363-d1398f5f82cc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.173492538Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-506a02c6-4fb9-4ffa-b363-d1398f5f82cc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.176346128Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.186837809Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-506a02c6-4fb9-4ffa-b363-d1398f5f82cc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.191964821Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/506a02c6-4fb9-4ffa-b363-d1398f5f82cc","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.19225493Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.192316492Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::19","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.192358034Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.192400215Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.195837043Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.199891392Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2025-03-13T18:27:24.303786673Z","hostname":"oxz_crucible_a9579ed0-0fba-47b4-8852-a9f9faa625fa","pid":22911,"uri":"/crucible/0/regions","method":"POST","req_id":"239822bf-5cb3-4119-be77-59329cd0b9b8","remote_addr":"[fd00:1122:3344:101::c]:45559","local_addr":"[fd00:1122:3344:101::19]:32345","component":"dropshot","latency_us":63,"response_code":"200"}