├── .gitignore ├── LICENSE.Berkeley ├── LICENSE.Cambridge ├── README.md ├── build.sbt ├── project └── plugins.sbt └── src ├── main └── scala │ ├── amoalu.scala │ ├── bram.scala │ ├── broadcast.scala │ ├── cache.scala │ ├── coherence.scala │ ├── consts.scala │ ├── converters.scala │ ├── directory.scala │ ├── dma.scala │ ├── ecc.scala │ ├── htif.scala │ ├── interconnect.scala │ ├── metadata.scala │ ├── network.scala │ ├── package.scala │ ├── prci.scala │ ├── rom.scala │ ├── rtc.scala │ ├── scr.scala │ ├── tag.scala │ ├── tagcache.scala │ ├── tilelink.scala │ ├── uncore.scala │ └── util.scala └── test └── verilog └── tagcache └── tagcache_tb.sv /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | project/target/ 3 | -------------------------------------------------------------------------------- /LICENSE.Berkeley: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2016, The Regents of the University of California 2 | (Regents). All Rights Reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 1. Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | 2. Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | 3. Neither the name of the Regents nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, 16 | SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING 17 | OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS 18 | BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 19 | 20 | REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 | PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED 23 | HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE 24 | MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 25 | -------------------------------------------------------------------------------- /LICENSE.Cambridge: -------------------------------------------------------------------------------- 1 | Copyright 2014-2017 University of Cambridge 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Uncore Library 2 | ============== 3 | 4 | This is the repository for uncore components assosciated with Rocket chip 5 | project. To uses these modules, include this repo as a git submodule within 6 | the your chip repository and add it as a project in your chip's build.scala. 7 | These components are only dependent on the ucb-bar/chisel repo, i.e. 8 | 9 | lazy val uncore = project.dependsOn(chisel) 10 | 11 | ScalaDoc for the uncore library is available here 12 | and an overview of the TileLink Protocol is available here, with associated CoherencePolicy documentation here. 13 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | organization := "edu.berkeley.cs" 2 | 3 | version := "2.0" 4 | 5 | name := "uncore" 6 | 7 | scalaVersion := "2.11.6" 8 | 9 | // Provide a managed dependency on X if -DXVersion="" is supplied on the command line. 10 | libraryDependencies ++= (Seq("chisel","junctions","cde").map { 11 | dep: String => sys.props.get(dep + "Version") map { "edu.berkeley.cs" %% dep % _ }}).flatten 12 | 13 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | resolvers += "jgit-repo" at "http://download.eclipse.org/jgit/maven" 2 | 3 | addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.5.3") 4 | 5 | addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "0.8.1") 6 | -------------------------------------------------------------------------------- /src/main/scala/amoalu.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | import Chisel._ 5 | import cde.{Parameters, Field} 6 | 7 | class StoreGen(typ: UInt, addr: UInt, dat: UInt, maxSize: Int) { 8 | val size = typ(log2Up(log2Up(maxSize)+1)-1,0) 9 | def misaligned = 10 | (addr & ((UInt(1) << size) - UInt(1))(log2Up(maxSize)-1,0)).orR 11 | 12 | def mask = { 13 | var res = UInt(1) 14 | for (i <- 0 until log2Up(maxSize)) { 15 | val upper = Mux(addr(i), res, UInt(0)) | Mux(size >= UInt(i+1), UInt((BigInt(1) << (1 << i))-1), UInt(0)) 16 | val lower = Mux(addr(i), UInt(0), res) 17 | res = Cat(upper, lower) 18 | } 19 | res 20 | } 21 | 22 | protected def genData(i: Int): UInt = 23 | if (i >= log2Up(maxSize)) dat 24 | else Mux(size === UInt(i), Fill(1 << (log2Up(maxSize)-i), dat((8 << i)-1,0)), genData(i+1)) 25 | 26 | def data = genData(0) 27 | def wordData = genData(2) 28 | } 29 | 30 | class StoreGenAligned(typ: UInt, addr: UInt, dat: UInt, maxSize: Int) extends StoreGen(typ, addr, dat, maxSize) { 31 | override def genData(i: Int) = dat 32 | } 33 | 34 | class LoadGen(typ: UInt, addr: UInt, dat: UInt, zero: Bool, maxSize: Int) { 35 | private val t = new StoreGen(typ, addr, dat, maxSize) 36 | private val signed = !typ(log2Up(log2Up(maxSize)+1)) 37 | 38 | private def genData(logMinSize: Int): UInt = { 39 | var res = dat 40 | for (i <- log2Up(maxSize)-1 to logMinSize by -1) { 41 | val pos = 8 << i 42 | val shifted = Mux(addr(i), res(2*pos-1,pos), res(pos-1,0)) 43 | val doZero = Bool(i == 0) && zero 44 | val zeroed = Mux(doZero, UInt(0), shifted) 45 | res = Cat(Mux(t.size === UInt(i) || doZero, Fill(8*maxSize-pos, signed && zeroed(pos-1)), res(8*maxSize-1,pos)), zeroed) 46 | } 47 | res 48 | } 49 | 50 | def wordData = genData(2) 51 | def data = genData(0) 52 | } 53 | 54 | class AMOALU(rhsIsAligned: Boolean = false)(implicit p: Parameters) extends CacheModule()(p) { 55 | val operandBits = p(AmoAluOperandBits) 56 | require(operandBits == 32 || operandBits == 64) 57 | val io = new Bundle { 58 | val addr = Bits(INPUT, blockOffBits) 59 | val cmd = Bits(INPUT, M_SZ) 60 | val typ = Bits(INPUT, MT_SZ) 61 | val lhs = Bits(INPUT, operandBits) 62 | val rhs = Bits(INPUT, operandBits) 63 | val out = Bits(OUTPUT, operandBits) 64 | } 65 | 66 | val storegen = 67 | if(rhsIsAligned) new StoreGenAligned(io.typ, io.addr, io.rhs, operandBits/8) 68 | else new StoreGen(io.typ, io.addr, io.rhs, operandBits/8) 69 | val rhs = storegen.wordData 70 | 71 | val sgned = io.cmd === M_XA_MIN || io.cmd === M_XA_MAX 72 | val max = io.cmd === M_XA_MAX || io.cmd === M_XA_MAXU 73 | val min = io.cmd === M_XA_MIN || io.cmd === M_XA_MINU 74 | val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization: 75 | io.typ === MT_B || io.typ === MT_BU 76 | 77 | val adder_out = 78 | if (operandBits == 32) io.lhs + rhs 79 | else { 80 | val mask = ~UInt(0,64) ^ (io.addr(2) << 31) 81 | (io.lhs & mask).toUInt + (rhs & mask) 82 | } 83 | 84 | val less = 85 | if (operandBits == 32) Mux(io.lhs(31) === rhs(31), io.lhs < rhs, Mux(sgned, io.lhs(31), io.rhs(31))) 86 | else { 87 | val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) 88 | val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63)) 89 | val lt_lo = io.lhs(31,0) < rhs(31,0) 90 | val lt_hi = io.lhs(63,32) < rhs(63,32) 91 | val eq_hi = io.lhs(63,32) === rhs(63,32) 92 | val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo) 93 | Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs)) 94 | } 95 | 96 | val out = Mux(io.cmd === M_XA_ADD, adder_out, 97 | Mux(io.cmd === M_XA_AND, io.lhs & rhs, 98 | Mux(io.cmd === M_XA_OR, io.lhs | rhs, 99 | Mux(io.cmd === M_XA_XOR, io.lhs ^ rhs, 100 | Mux(Mux(less, min, max), io.lhs, 101 | storegen.data))))) 102 | 103 | val wmask = FillInterleaved(8, storegen.mask) 104 | io.out := wmask & out | ~wmask & io.lhs 105 | } 106 | -------------------------------------------------------------------------------- /src/main/scala/bram.scala: -------------------------------------------------------------------------------- 1 | package uncore 2 | 3 | import Chisel._ 4 | import cde.{Parameters, Field} 5 | import junctions._ 6 | 7 | class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module 8 | with HasTileLinkParameters { 9 | val io = new ClientUncachedTileLinkIO().flip 10 | 11 | val bram = SeqMem(depth, Bits(width = tlDataBits)) 12 | 13 | val (s0_get :: s0_getblk :: s0_put :: s0_putblk :: Nil) = Seq( 14 | Acquire.getType, Acquire.getBlockType, Acquire.putType, Acquire.putBlockType 15 | ).map(io.acquire.bits.isBuiltInType _) 16 | 17 | val fire_acq = io.acquire.fire() 18 | val fire_gnt = io.grant.fire() 19 | 20 | val multibeat = Reg(init = Bool(false)) 21 | when (fire_acq) { 22 | multibeat := s0_getblk 23 | } 24 | 25 | val s0_valid = io.acquire.valid || multibeat 26 | val s1_valid = Reg(next = s0_valid, init = Bool(false)) 27 | val s1_acq = RegEnable(io.acquire.bits, fire_acq) 28 | 29 | val s0_addr = Cat(io.acquire.bits.addr_block, io.acquire.bits.addr_beat) 30 | val s1_beat = s1_acq.addr_beat + Mux(io.grant.ready, UInt(1), UInt(0)) 31 | val s1_addr = Cat(s1_acq.addr_block, s1_beat) 32 | val raddr = Mux(multibeat, s1_addr, s0_addr) 33 | 34 | val last = (s1_acq.addr_beat === UInt(tlDataBeats-1)) 35 | val ren = (io.acquire.valid && (s0_get || s0_getblk)) || (multibeat && !last) 36 | val wen = (io.acquire.valid && (s0_put || s0_putblk)) 37 | 38 | val rdata = bram.read(raddr, ren) 39 | val wdata = io.acquire.bits.data 40 | val wmask = io.acquire.bits.wmask() 41 | assert(!io.acquire.valid || wmask === Fill(tlDataBytes, Bool(true)), 42 | "bram: subblock writes not supported") 43 | when (wen) { 44 | bram.write(s0_addr, wdata) 45 | } 46 | 47 | when (multibeat && fire_gnt) { 48 | s1_acq.addr_beat := s1_beat 49 | when (last) { 50 | multibeat := Bool(false) 51 | } 52 | } 53 | 54 | io.grant.valid := s1_valid 55 | io.grant.bits := Grant( 56 | is_builtin_type = Bool(true), 57 | g_type = s1_acq.getBuiltInGrantType(), 58 | client_xact_id = s1_acq.client_xact_id, 59 | manager_xact_id = UInt(0), 60 | addr_beat = s1_acq.addr_beat, 61 | data = rdata) 62 | 63 | val stall = multibeat || (io.grant.valid && !io.grant.ready) 64 | io.acquire.ready := !stall 65 | } 66 | 67 | class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { 68 | val io = new HastiSlaveIO 69 | 70 | val hastiDataBytes = hastiDataBits/8 71 | 72 | val wdata = Vec.tabulate(hastiDataBytes)(i => io.hwdata(8*(i+1)-1,8*i)) 73 | val waddr = Reg(UInt(width = hastiAddrBits)) 74 | val wvalid = Reg(init = Bool(false)) 75 | val wsize = Reg(UInt(width = SZ_HSIZE)) 76 | val ram = SeqMem(depth, Vec(hastiDataBytes, Bits(width = 8))) 77 | 78 | val max_wsize = log2Ceil(hastiDataBytes) 79 | val wmask_lut = MuxLookup(wsize, SInt(-1, hastiDataBytes).asUInt, 80 | (0 until max_wsize).map(1 << _).map(sz => (UInt(sz) -> UInt((1 << sz << sz) - 1)))) 81 | val wmask = (wmask_lut << waddr(max_wsize - 1, 0))(hastiDataBytes - 1, 0) 82 | 83 | val is_trans = io.hsel && (io.htrans === HTRANS_NONSEQ || io.htrans === HTRANS_SEQ) 84 | val raddr = io.haddr >> UInt(2) 85 | val ren = is_trans && !io.hwrite 86 | val bypass = Reg(init = Bool(false)) 87 | val last_wdata = Reg(next = wdata) 88 | val last_wmask = Reg(next = wmask) 89 | 90 | when (is_trans && io.hwrite) { 91 | waddr := io.haddr 92 | wsize := io.hsize 93 | wvalid := Bool(true) 94 | } .otherwise { wvalid := Bool(false) } 95 | 96 | when (ren) { bypass := wvalid && (waddr >> UInt(2)) === raddr } 97 | 98 | when (wvalid) { 99 | ram.write(waddr >> UInt(2), wdata, wmask.toBools) 100 | } 101 | 102 | val rdata = ram.read(raddr, ren) 103 | io.hrdata := Cat(rdata.zip(wmask.toBools).zip(wdata).map { 104 | case ((rbyte, wsel), wbyte) => Mux(wsel && bypass, wbyte, rbyte) 105 | }.reverse) 106 | 107 | io.hreadyout := Bool(true) 108 | io.hresp := HRESP_OKAY 109 | } 110 | -------------------------------------------------------------------------------- /src/main/scala/broadcast.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | import Chisel._ 5 | import cde.{Parameters, Field} 6 | 7 | case object L2StoreDataQueueDepth extends Field[Int] 8 | 9 | trait HasBroadcastHubParameters extends HasCoherenceAgentParameters { 10 | val sdqDepth = p(L2StoreDataQueueDepth)*innerDataBeats 11 | val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) 12 | val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases 13 | } 14 | 15 | class DataQueueLocation(implicit p: Parameters) extends CoherenceAgentBundle()(p) 16 | with HasBroadcastHubParameters { 17 | val idx = UInt(width = dqIdxBits) 18 | val loc = UInt(width = log2Ceil(nDataQueueLocations)) 19 | } 20 | 21 | object DataQueueLocation { 22 | def apply(idx: UInt, loc: UInt)(implicit p: Parameters) = { 23 | val d = Wire(new DataQueueLocation) 24 | d.idx := idx 25 | d.loc := loc 26 | d 27 | } 28 | } 29 | 30 | class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) 31 | with HasBroadcastHubParameters { 32 | val internalDataBits = new DataQueueLocation().getWidth 33 | val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) 34 | 35 | val usingStoreDataQueue = p.alterPartial({ 36 | case TLKey(`innerTLId`) => innerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits)) 37 | case TLKey(`outerTLId`) => outerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits)) 38 | }) 39 | 40 | // Create SHRs for outstanding transactions 41 | val trackerList = 42 | (0 until nReleaseTransactors).map(id => 43 | Module(new BroadcastVoluntaryReleaseTracker(id)(usingStoreDataQueue))) ++ 44 | (nReleaseTransactors until nTransactors).map(id => 45 | Module(new BroadcastAcquireTracker(id)(usingStoreDataQueue))) 46 | 47 | // Propagate incoherence flags 48 | trackerList.map(_.io.incoherent := io.incoherent) 49 | 50 | // Queue to store impending Put data 51 | val sdq_data = Reg(Vec(sdqDepth, io.iacq().data)) 52 | val sdq_tag = Reg(Vec(sdqDepth, io.iacq().tag)) 53 | val sdq_val = Reg(init=Bits(0, sdqDepth)) 54 | val sdq_alloc_id = PriorityEncoder(~sdq_val) 55 | val sdq_rdy = !sdq_val.andR 56 | val sdq_enq = trackerList.map( t => 57 | (t.io.alloc.iacq || t.io.matches.iacq) && 58 | t.io.inner.acquire.fire() && 59 | t.io.iacq().hasData() 60 | ).reduce(_||_) 61 | when (sdq_enq) { 62 | sdq_data(sdq_alloc_id) := io.iacq().data 63 | sdq_tag(sdq_alloc_id) := io.iacq().tag 64 | } 65 | 66 | // Handle acquire transaction initiation 67 | val irel_vs_iacq_conflict = 68 | io.inner.acquire.valid && 69 | io.inner.release.valid && 70 | io.irel().conflicts(io.iacq()) 71 | val sdqLoc = List.fill(nTransactors) { 72 | DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits 73 | } 74 | doInputRoutingWithAllocation( 75 | io.inner.acquire, 76 | trackerList.map(_.io.inner.acquire), 77 | trackerList.map(_.io.matches.iacq), 78 | trackerList.map(_.io.alloc.iacq), 79 | Some(sdqLoc), 80 | Some(sdq_rdy && !irel_vs_iacq_conflict), 81 | Some(sdq_rdy)) 82 | 83 | // Queue to store impending Voluntary Release data 84 | val voluntary = io.irel().isVoluntary() 85 | val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData() 86 | val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width 87 | val vwbdq_data = Reg(Vec(innerDataBeats, io.irel().data)) //TODO Assumes nReleaseTransactors == 1 88 | val vwbdq_tag = Reg(Vec(innerDataBeats, io.irel().tag)) //TODO Assumes nReleaseTransactors == 1 89 | when(vwbdq_enq) { 90 | vwbdq_data(rel_data_cnt) := io.irel().data 91 | vwbdq_tag(rel_data_cnt) := io.irel().tag 92 | } 93 | 94 | // Handle releases, which might be voluntary and might have data 95 | val vwbqLoc = (0 until nTransactors).map(i => 96 | (DataQueueLocation(rel_data_cnt, 97 | (if(i < nReleaseTransactors) inVolWBQueue 98 | else inClientReleaseQueue)).toBits)) 99 | doInputRoutingWithAllocation( 100 | io.inner.release, 101 | trackerList.map(_.io.inner.release), 102 | trackerList.map(_.io.matches.irel), 103 | trackerList.map(_.io.alloc.irel), 104 | Some(vwbqLoc)) 105 | 106 | // Wire probe requests and grant reply to clients, finish acks from clients 107 | // Note that we bypass the Grant data subbundles 108 | doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) 109 | io.inner.grant.bits.data := io.outer.grant.bits.data 110 | io.inner.grant.bits.tag := io.outer.grant.bits.tag 111 | io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat 112 | doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) 113 | doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) 114 | 115 | // Create an arbiter for the one memory port 116 | val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size) 117 | (usingStoreDataQueue.alterPartial({ case TLId => p(OuterTLId) }))) 118 | outer_arb.io.in <> trackerList.map(_.io.outer) 119 | // Get the pending data out of the store data queue 120 | val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) 121 | val is_in_sdq = outer_data_ptr.loc === inStoreQueue 122 | val free_sdq = io.outer.acquire.fire() && 123 | io.outer.acquire.bits.hasData() && 124 | outer_data_ptr.loc === inStoreQueue 125 | io.outer <> outer_arb.io.out 126 | io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( 127 | inStoreQueue -> sdq_data(outer_data_ptr.idx), 128 | inVolWBQueue -> vwbdq_data(outer_data_ptr.idx))) 129 | io.outer.acquire.bits.tag := MuxLookup(outer_data_ptr.loc, io.irel().tag, Array( 130 | inStoreQueue -> sdq_tag(outer_data_ptr.idx), 131 | inVolWBQueue -> vwbdq_tag(outer_data_ptr.idx))) 132 | 133 | // Update SDQ valid bits 134 | when (io.outer.acquire.valid || sdq_enq) { 135 | sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | 136 | PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) 137 | } 138 | } 139 | 140 | class BroadcastXactTracker(implicit p: Parameters) extends XactTracker()(p) { 141 | val io = new ManagerXactTrackerIO 142 | pinAllReadyValidLow(io) 143 | } 144 | 145 | class BroadcastVoluntaryReleaseTracker(trackerId: Int) 146 | (implicit p: Parameters) extends BroadcastXactTracker()(p) { 147 | val s_idle :: s_busy :: Nil = Enum(UInt(), 2) 148 | val state = Reg(init=s_idle) 149 | 150 | val xact = Reg(new BufferedReleaseFromSrc()(p.alterPartial({ case TLId => innerTLId }))) 151 | val coh = ManagerMetadata.onReset 152 | 153 | val pending_irels = Reg(init=Bits(0, width = io.inner.tlDataBeats)) 154 | val pending_writes = Reg(init=Bits(0, width = io.outer.tlDataBeats)) 155 | val pending_ignt = Reg(init=Bool(false)) 156 | 157 | val all_pending_done = !(pending_irels.orR || pending_writes.orR || pending_ignt) 158 | 159 | // Accept a voluntary Release (and any further beats of data) 160 | pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) 161 | io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irels.orR 162 | when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data } 163 | 164 | // Write the voluntarily written back data to outer memory using an Acquire.PutBlock 165 | //TODO: Use io.outer.release instead? 166 | pending_writes := (pending_writes & dropPendingBitWhenBeatHasData(io.outer.acquire)) | 167 | addPendingBitWhenBeatHasData(io.inner.release) 168 | val curr_write_beat = PriorityEncoder(pending_writes) 169 | io.outer.acquire.valid := state === s_busy && pending_writes.orR 170 | io.outer.acquire.bits := PutBlock( 171 | client_xact_id = UInt(trackerId), 172 | addr_block = xact.addr_block, 173 | addr_beat = curr_write_beat, 174 | data = xact.data_buffer(curr_write_beat)) 175 | (p.alterPartial({ case TLId => outerTLId })) 176 | 177 | // Send an acknowledgement 178 | io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels && io.outer.grant.valid 179 | io.inner.grant.bits := coh.makeGrant(xact) 180 | when(io.inner.grant.fire()) { pending_ignt := Bool(false) } 181 | io.outer.grant.ready := state === s_busy && io.inner.grant.ready 182 | 183 | // State machine updates and transaction handler metadata intialization 184 | when(state === s_idle && io.inner.release.valid && io.alloc.irel) { 185 | xact := io.irel() 186 | when(io.irel().hasMultibeatData()) { 187 | pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) 188 | }. otherwise { 189 | pending_irels := UInt(0) 190 | } 191 | pending_writes := addPendingBitWhenBeatHasData(io.inner.release) 192 | pending_ignt := io.irel().requiresAck() 193 | state := s_busy 194 | } 195 | when(state === s_busy && all_pending_done) { state := s_idle } 196 | 197 | // These IOs are used for routing in the parent 198 | io.matches.iacq := (state =/= s_idle) && xact.conflicts(io.iacq()) 199 | io.matches.irel := (state =/= s_idle) && xact.conflicts(io.irel()) && io.irel().isVoluntary() 200 | io.matches.oprb := Bool(false) 201 | 202 | // Checks for illegal behavior 203 | assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()), 204 | "VoluntaryReleaseTracker accepted Release that wasn't voluntary!") 205 | } 206 | 207 | class BroadcastAcquireTracker(trackerId: Int) 208 | (implicit p: Parameters) extends BroadcastXactTracker()(p) { 209 | val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) 210 | val state = Reg(init=s_idle) 211 | 212 | val xact = Reg(new BufferedAcquireFromSrc()(p.alterPartial({ case TLId => innerTLId }))) 213 | val coh = ManagerMetadata.onReset 214 | 215 | assert(!(state =/= s_idle && xact.isBuiltInType() && 216 | Vec(Acquire.putAtomicType, Acquire.getPrefetchType, Acquire.putPrefetchType).contains(xact.a_type)), 217 | "Broadcast Hub does not support PutAtomics or prefetches") // TODO 218 | 219 | val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) 220 | val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCachingClients)) 221 | val curr_p_id = PriorityEncoder(pending_probes) 222 | val mask_self = SInt(-1, width = io.inner.tlNCachingClients) 223 | .toUInt 224 | .bitSet(io.inner.acquire.bits.client_id, io.inner.acquire.bits.requiresSelfProbe()) 225 | val mask_incoherent = mask_self & ~io.incoherent.toBits 226 | 227 | val collect_iacq_data = Reg(init=Bool(false)) 228 | val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) 229 | val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire, io.alloc.iacq || collect_iacq_data) 230 | val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) 231 | val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant) 232 | val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) 233 | val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) 234 | val pending_ognt_ack = Reg(init=Bool(false)) 235 | val pending_outer_write = xact.hasData() 236 | val pending_outer_write_ = io.iacq().hasData() 237 | val pending_outer_read = io.ignt().hasData() 238 | val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() 239 | val subblock_type = xact.isSubBlockType() 240 | 241 | // These IOs are used for routing in the parent 242 | io.matches.iacq := (state =/= s_idle) && xact.conflicts(io.iacq()) 243 | io.matches.irel := (state =/= s_idle) && xact.conflicts(io.irel()) && !io.irel().isVoluntary() 244 | io.matches.oprb := Bool(false) 245 | 246 | val outerParams = p.alterPartial({ case TLId => outerTLId }) 247 | 248 | val oacq_probe = PutBlock( 249 | client_xact_id = UInt(trackerId), 250 | addr_block = io.irel().addr_block, 251 | addr_beat = io.irel().addr_beat, 252 | data = io.irel().data)(outerParams) 253 | 254 | val oacq_write_beat = Put( 255 | client_xact_id = UInt(trackerId), 256 | addr_block = xact.addr_block, 257 | addr_beat = xact.addr_beat, 258 | data = xact.data_buffer(0), 259 | wmask = xact.wmask(), 260 | tmask = xact.tmask())(outerParams) 261 | 262 | val oacq_write_block = PutBlock( 263 | client_xact_id = UInt(trackerId), 264 | addr_block = xact.addr_block, 265 | addr_beat = oacq_data_cnt, 266 | data = xact.data_buffer(oacq_data_cnt), 267 | wmask = xact.wmask_buffer(oacq_data_cnt), 268 | tmask = xact.tmask_buffer(oacq_data_cnt))(outerParams) 269 | 270 | val oacq_read_beat = Get( 271 | client_xact_id = UInt(trackerId), 272 | addr_block = xact.addr_block, 273 | addr_beat = xact.addr_beat, 274 | addr_byte = xact.addr_byte(), 275 | operand_size = xact.op_size(), 276 | alloc = Bool(false))(outerParams) 277 | 278 | val oacq_read_block = GetBlock( 279 | client_xact_id = UInt(trackerId), 280 | addr_block = xact.addr_block)(outerParams) 281 | 282 | io.outer.acquire.valid := Bool(false) 283 | 284 | // Something is SERIOUSLY WRONG this version of Chisel. 285 | // The following code emits WRONG verilog. 286 | 287 | // io.outer.acquire.bits := Mux(state === s_probe, oacq_probe, 288 | // Mux(state === s_mem_write, 289 | // Mux(subblock_type, oacq_write_beat, oacq_write_block), 290 | // Mux(subblock_type, oacq_read_beat, oacq_read_block))) 291 | 292 | io.outer.acquire.bits := oacq_probe 293 | when(state === s_probe) { 294 | io.outer.acquire.bits := oacq_probe 295 | } 296 | when(state === s_mem_write && subblock_type) { 297 | io.outer.acquire.bits := oacq_write_beat 298 | } 299 | when(state === s_mem_write && !subblock_type) { 300 | io.outer.acquire.bits := oacq_write_block 301 | } 302 | when(state === s_mem_read && subblock_type) { 303 | io.outer.acquire.bits := oacq_read_beat 304 | } 305 | when(state === s_mem_read && !subblock_type) { 306 | io.outer.acquire.bits := oacq_read_block 307 | } 308 | 309 | io.outer.grant.ready := Bool(false) 310 | 311 | io.inner.probe.valid := Bool(false) 312 | io.inner.probe.bits := coh.makeProbe(curr_p_id, xact) 313 | 314 | io.inner.grant.valid := Bool(false) 315 | io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent 316 | 317 | io.inner.acquire.ready := Bool(false) 318 | io.inner.release.ready := Bool(false) 319 | io.inner.finish.ready := Bool(false) 320 | 321 | assert(!(state =/= s_idle && collect_iacq_data && io.inner.acquire.fire() && 322 | io.iacq().client_id =/= xact.client_id), 323 | "AcquireTracker accepted data beat from different network source than initial request.") 324 | 325 | assert(!(state =/= s_idle && collect_iacq_data && io.inner.acquire.fire() && 326 | io.iacq().client_xact_id =/= xact.client_xact_id), 327 | "AcquireTracker accepted data beat from different client transaction than initial request.") 328 | 329 | assert(!(state === s_idle && io.inner.acquire.fire() && io.alloc.iacq && 330 | io.iacq().hasMultibeatData() && io.iacq().addr_beat =/= UInt(0)), 331 | "AcquireTracker initialized with a tail data beat.") 332 | 333 | when(collect_iacq_data) { 334 | io.inner.acquire.ready := Bool(true) 335 | when(io.inner.acquire.valid) { 336 | xact.data_buffer(io.iacq().addr_beat) := io.iacq().data 337 | xact.wmask_buffer(io.iacq().addr_beat) := io.iacq().wmask() 338 | xact.tmask_buffer(io.iacq().addr_beat) := io.iacq().tmask() 339 | iacq_data_valid := iacq_data_valid.bitSet(io.iacq().addr_beat, Bool(true)) 340 | } 341 | when(iacq_data_done) { collect_iacq_data := Bool(false) } 342 | } 343 | 344 | when(pending_ognt_ack) { 345 | io.outer.grant.ready := Bool(true) 346 | when(io.outer.grant.valid) { pending_ognt_ack := Bool(false) } 347 | //TODO add finish queue if this isnt the last level manager 348 | } 349 | 350 | switch (state) { 351 | is(s_idle) { 352 | io.inner.acquire.ready := Bool(true) 353 | when(io.inner.acquire.valid && io.alloc.iacq) { 354 | xact := io.iacq() 355 | xact.data_buffer(UInt(0)) := io.iacq().data 356 | xact.wmask_buffer(UInt(0)) := io.iacq().wmask() 357 | xact.tmask_buffer(UInt(0)) := io.iacq().tmask() 358 | collect_iacq_data := io.iacq().hasMultibeatData() 359 | iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat 360 | val needs_probes = mask_incoherent.orR 361 | when(needs_probes) { 362 | pending_probes := mask_incoherent 363 | release_count := PopCount(mask_incoherent) 364 | } 365 | state := Mux(needs_probes, s_probe, 366 | Mux(pending_outer_write_, s_mem_write, 367 | Mux(pending_outer_read_, s_mem_read, s_make_grant))) 368 | } 369 | } 370 | is(s_probe) { 371 | // Generate probes 372 | io.inner.probe.valid := pending_probes.orR 373 | when(io.inner.probe.ready) { 374 | pending_probes := pending_probes & ~UIntToOH(curr_p_id) 375 | } 376 | 377 | // Handle releases, which may have data to be written back 378 | val matches = io.matches.irel 379 | io.inner.release.ready := (!io.irel().hasData() || io.outer.acquire.ready) && matches 380 | when(io.inner.release.valid && matches) { 381 | when(io.irel().hasData()) { 382 | io.outer.acquire.valid := Bool(true) 383 | when(io.outer.acquire.ready) { 384 | when(oacq_data_done) { 385 | pending_ognt_ack := Bool(true) 386 | release_count := release_count - UInt(1) 387 | when(release_count === UInt(1)) { 388 | state := Mux(pending_outer_write, s_mem_write, 389 | Mux(pending_outer_read, s_mem_read, s_make_grant)) 390 | } 391 | } 392 | } 393 | } .otherwise { 394 | release_count := release_count - UInt(1) 395 | when(release_count === UInt(1)) { 396 | state := Mux(pending_outer_write, s_mem_write, 397 | Mux(pending_outer_read, s_mem_read, s_make_grant)) 398 | } 399 | } 400 | } 401 | } 402 | is(s_mem_write) { // Write data to outer memory 403 | io.outer.acquire.valid := !pending_ognt_ack && (!collect_iacq_data || iacq_data_valid(oacq_data_cnt)) 404 | when(oacq_data_done) { 405 | pending_ognt_ack := Bool(true) 406 | state := Mux(pending_outer_read, s_mem_read, s_mem_resp) 407 | } 408 | } 409 | is(s_mem_read) { // Read data from outer memory (possibly what was just written) 410 | io.outer.acquire.valid := !pending_ognt_ack 411 | when(io.outer.acquire.fire()) { state := s_mem_resp } 412 | } 413 | is(s_mem_resp) { // Wait to forward grants from outer memory 414 | io.outer.grant.ready := io.inner.grant.ready 415 | io.inner.grant.valid := io.outer.grant.valid 416 | when(ignt_data_done) { 417 | state := Mux(io.ignt().requiresAck(), s_ack, s_idle) 418 | } 419 | } 420 | is(s_make_grant) { // Manufacture a local grant (some kind of permission upgrade) 421 | io.inner.grant.valid := Bool(true) 422 | when(io.inner.grant.ready) { 423 | state := Mux(io.ignt().requiresAck(), s_ack, s_idle) 424 | } 425 | } 426 | is(s_ack) { // Wait for transaction to complete 427 | io.inner.finish.ready := Bool(true) 428 | when(io.inner.finish.valid) { state := s_idle } 429 | } 430 | } 431 | } 432 | -------------------------------------------------------------------------------- /src/main/scala/coherence.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | import Chisel._ 5 | 6 | /** The entire CoherencePolicy API consists of the following three traits: 7 | * HasCustomTileLinkMessageTypes, used to define custom messages 8 | * HasClientSideCoherencePolicy, for client coherence agents 9 | * HasManagerSideCoherencePolicy, for manager coherence agents 10 | */ 11 | abstract class CoherencePolicy(val dir: DirectoryRepresentation) 12 | extends HasCustomTileLinkMessageTypes 13 | with HasClientSideCoherencePolicy 14 | with HasManagerSideCoherencePolicy 15 | 16 | /** This API defines the custom, coherence-policy-defined message types, 17 | * as opposed to the built-in ones found in tilelink.scala. 18 | * Policies must enumerate the custom messages to be sent over each 19 | * channel, as well as which of them have associated data. 20 | */ 21 | trait HasCustomTileLinkMessageTypes { 22 | val nAcquireTypes: Int 23 | def acquireTypeWidth = log2Up(nAcquireTypes) 24 | val nProbeTypes: Int 25 | def probeTypeWidth = log2Up(nProbeTypes) 26 | val nReleaseTypes: Int 27 | def releaseTypeWidth = log2Up(nReleaseTypes) 28 | val nGrantTypes: Int 29 | def grantTypeWidth = log2Up(nGrantTypes) 30 | 31 | val acquireTypesWithData = Nil // Only built-in Acquire types have data for now 32 | def releaseTypesWithData: Vec[UInt] 33 | def grantTypesWithData: Vec[UInt] 34 | } 35 | 36 | /** This API contains all functions required for client coherence agents. 37 | * Policies must enumerate the number of client states and define their 38 | * permissions with respect to memory operations. Policies must fill in functions 39 | * to control which messages are sent and how metadata is updated in response 40 | * to coherence events. These funtions are generally called from within the 41 | * ClientMetadata class in metadata.scala 42 | */ 43 | trait HasClientSideCoherencePolicy { 44 | // Client coherence states and their permissions 45 | val nClientStates: Int 46 | def clientStateWidth = log2Ceil(nClientStates) 47 | def clientStatesWithReadPermission: Vec[UInt] 48 | def clientStatesWithWritePermission: Vec[UInt] 49 | def clientStatesWithDirtyData: Vec[UInt] 50 | 51 | // Transaction initiation logic 52 | def isValid(meta: ClientMetadata): Bool 53 | def isHit(cmd: UInt, meta: ClientMetadata): Bool = { 54 | Mux(isWriteIntent(cmd), 55 | clientStatesWithWritePermission.contains(meta.state), 56 | clientStatesWithReadPermission.contains(meta.state)) 57 | } 58 | //TODO: Assumes all states with write permissions also have read permissions 59 | def requiresAcquireOnSecondaryMiss( 60 | first_cmd: UInt, 61 | second_cmd: UInt, 62 | meta: ClientMetadata): Bool = { 63 | isWriteIntent(second_cmd) && !isWriteIntent(first_cmd) 64 | } 65 | //TODO: Assumes all cache ctrl ops writeback dirty data, and 66 | // doesn't issue transaction when e.g. downgrading Exclusive to Shared: 67 | def requiresReleaseOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = 68 | clientStatesWithDirtyData.contains(meta.state) 69 | 70 | // Determine which custom message type to use 71 | def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt 72 | def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt 73 | def getReleaseType(p: HasProbeType, meta: ClientMetadata): UInt 74 | 75 | // Mutate ClientMetadata based on messages or cmds 76 | def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata 77 | def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata 78 | def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata): ClientMetadata 79 | def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata): ClientMetadata 80 | } 81 | 82 | /** This API contains all functions required for manager coherence agents. 83 | * Policies must enumerate the number of manager states. Policies must fill 84 | * in functions to control which Probe and Grant messages are sent and how 85 | * metadata should be updated in response to coherence events. These funtions 86 | * are generally called from within the ManagerMetadata class in metadata.scala 87 | */ 88 | trait HasManagerSideCoherencePolicy extends HasDirectoryRepresentation { 89 | val nManagerStates: Int 90 | def masterStateWidth = log2Ceil(nManagerStates) 91 | 92 | // Transaction probing logic 93 | def requiresProbes(acq: HasAcquireType, meta: ManagerMetadata): Bool 94 | def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool 95 | 96 | // Determine which custom message type to use in response 97 | def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt 98 | def getProbeType(acq: HasAcquireType, meta: ManagerMetadata): UInt 99 | def getGrantType(acq: HasAcquireType, meta: ManagerMetadata): UInt 100 | def getExclusiveGrantType(): UInt 101 | 102 | // Mutate ManagerMetadata based on messages or cmds 103 | def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata): ManagerMetadata 104 | def managerMetadataOnGrant(outgoing: HasGrantType, dst: UInt, meta: ManagerMetadata) = 105 | ManagerMetadata(sharers=Mux(outgoing.isBuiltInType(), // Assumes all built-ins are uncached 106 | meta.sharers, 107 | dir.push(meta.sharers, dst)))(meta.p) 108 | //state = meta.state) TODO: Fix 0-width wires in Chisel 109 | } 110 | 111 | /** The following concrete implementations of CoherencePolicy each provide the 112 | * functionality of one particular protocol. 113 | */ 114 | 115 | /** A simple protocol with only two Client states. 116 | * Data is always assumed to be dirty. 117 | * Only a single client may ever have a copy of a block at a time. 118 | */ 119 | class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { 120 | // Message types 121 | val nAcquireTypes = 1 122 | val nProbeTypes = 2 123 | val nReleaseTypes = 4 124 | val nGrantTypes = 1 125 | 126 | val acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) 127 | val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) 128 | val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) 129 | val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) 130 | 131 | def releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) 132 | def grantTypesWithData = Vec(grantExclusive) 133 | 134 | // Client states and functions 135 | val nClientStates = 2 136 | val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) 137 | 138 | def clientStatesWithReadPermission = Vec(clientValid) 139 | def clientStatesWithWritePermission = Vec(clientValid) 140 | def clientStatesWithDirtyData = Vec(clientValid) 141 | 142 | def isValid (meta: ClientMetadata): Bool = meta.state =/= clientInvalid 143 | 144 | def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive 145 | 146 | def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { 147 | val dirty = clientStatesWithDirtyData.contains(meta.state) 148 | MuxLookup(cmd, releaseCopyAck, Array( 149 | M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), 150 | M_PRODUCE -> Mux(dirty, releaseCopyData, releaseCopyAck), 151 | M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) 152 | } 153 | 154 | def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = 155 | MuxLookup(incoming.p_type, releaseInvalidateAck, Array( 156 | probeInvalidate -> getReleaseType(M_FLUSH, meta), 157 | probeCopy -> getReleaseType(M_CLEAN, meta))) 158 | 159 | def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta 160 | 161 | def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = 162 | ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state))(meta.p) 163 | 164 | def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata) = 165 | ClientMetadata(Mux(incoming.isBuiltInType(), clientInvalid, clientValid))(meta.p) 166 | 167 | def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = 168 | ClientMetadata(Mux(incoming.p_type === probeInvalidate, 169 | clientInvalid, meta.state))(meta.p) 170 | 171 | // Manager states and functions: 172 | val nManagerStates = 0 // We don't actually need any states for this protocol 173 | 174 | def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = !dir.none(meta.sharers) 175 | 176 | def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) 177 | 178 | def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = 179 | MuxLookup(cmd, probeCopy, Array( 180 | M_FLUSH -> probeInvalidate)) 181 | 182 | def getProbeType(a: HasAcquireType, meta: ManagerMetadata): UInt = 183 | Mux(a.isBuiltInType(), 184 | MuxLookup(a.a_type, probeCopy, Array( 185 | Acquire.getBlockType -> probeCopy, 186 | Acquire.putBlockType -> probeInvalidate, 187 | Acquire.getType -> probeCopy, 188 | Acquire.putType -> probeInvalidate, 189 | Acquire.getPrefetchType -> probeCopy, 190 | Acquire.putPrefetchType -> probeInvalidate, 191 | Acquire.putAtomicType -> probeInvalidate)), 192 | probeInvalidate) 193 | 194 | def getGrantType(a: HasAcquireType, meta: ManagerMetadata): UInt = 195 | Mux(a.isBuiltInType(), Acquire.getBuiltInGrantType(a.a_type), grantExclusive) 196 | def getExclusiveGrantType(): UInt = grantExclusive 197 | 198 | def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { 199 | val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) 200 | MuxBundle(meta, Array( 201 | incoming.is(releaseInvalidateData) -> popped, 202 | incoming.is(releaseInvalidateAck) -> popped)) 203 | } 204 | } 205 | 206 | /** A simple protocol with only three Client states. 207 | * Data is marked as dirty when written. 208 | * Only a single client may ever have a copy of a block at a time. 209 | */ 210 | class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { 211 | // Message types 212 | val nAcquireTypes = 2 213 | val nProbeTypes = 3 214 | val nReleaseTypes = 6 215 | val nGrantTypes = 1 216 | 217 | val acquireExclusiveClean :: acquireExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) 218 | val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) 219 | val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) 220 | val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) 221 | 222 | def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) 223 | def grantTypesWithData = Vec(grantExclusive) 224 | 225 | // Client states and functions 226 | val nClientStates = 3 227 | val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) 228 | 229 | def clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) 230 | def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) 231 | def clientStatesWithDirtyData = Vec(clientExclusiveDirty) 232 | 233 | def isValid (meta: ClientMetadata) = meta.state =/= clientInvalid 234 | 235 | def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = 236 | Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) 237 | 238 | def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { 239 | val dirty = clientStatesWithDirtyData.contains(meta.state) 240 | MuxLookup(cmd, releaseCopyAck, Array( 241 | M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), 242 | M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), 243 | M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) 244 | } 245 | 246 | def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = 247 | MuxLookup(incoming.p_type, releaseInvalidateAck, Array( 248 | probeInvalidate -> getReleaseType(M_FLUSH, meta), 249 | probeDowngrade -> getReleaseType(M_PRODUCE, meta), 250 | probeCopy -> getReleaseType(M_CLEAN, meta))) 251 | 252 | def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = 253 | ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state))(meta.p) 254 | 255 | def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = 256 | ClientMetadata( 257 | MuxLookup(cmd, meta.state, Array( 258 | M_FLUSH -> clientInvalid, 259 | M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state))))(meta.p) 260 | 261 | def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata) = 262 | ClientMetadata( 263 | Mux(incoming.isBuiltInType(), clientInvalid, 264 | Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean)))(meta.p) 265 | 266 | def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = 267 | ClientMetadata( 268 | MuxLookup(incoming.p_type, meta.state, Array( 269 | probeInvalidate -> clientInvalid, 270 | probeDowngrade -> clientExclusiveClean, 271 | probeCopy -> meta.state)))(meta.p) 272 | 273 | // Manager states and functions: 274 | val nManagerStates = 0 // We don't actually need any states for this protocol 275 | 276 | def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = !dir.none(meta.sharers) 277 | def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) 278 | 279 | def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = 280 | MuxLookup(cmd, probeCopy, Array( 281 | M_FLUSH -> probeInvalidate, 282 | M_PRODUCE -> probeDowngrade)) 283 | 284 | def getProbeType(a: HasAcquireType, meta: ManagerMetadata): UInt = 285 | Mux(a.isBuiltInType(), 286 | MuxLookup(a.a_type, probeCopy, Array( 287 | Acquire.getBlockType -> probeCopy, 288 | Acquire.putBlockType -> probeInvalidate, 289 | Acquire.getType -> probeCopy, 290 | Acquire.putType -> probeInvalidate, 291 | Acquire.getPrefetchType -> probeCopy, 292 | Acquire.putPrefetchType -> probeInvalidate, 293 | Acquire.putAtomicType -> probeInvalidate)), 294 | probeInvalidate) 295 | 296 | def getGrantType(a: HasAcquireType, meta: ManagerMetadata): UInt = 297 | Mux(a.isBuiltInType(), Acquire.getBuiltInGrantType(a.a_type), grantExclusive) 298 | def getExclusiveGrantType(): UInt = grantExclusive 299 | 300 | def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { 301 | val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) 302 | MuxBundle(meta, Array( 303 | incoming.is(releaseInvalidateData) -> popped, 304 | incoming.is(releaseInvalidateAck) -> popped)) 305 | } 306 | } 307 | 308 | /** A protocol with only three Client states. 309 | * Data is always assumed to be dirty. 310 | * Multiple clients may share read permissions on a block at the same time. 311 | */ 312 | class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { 313 | // Message types 314 | val nAcquireTypes = 2 315 | val nProbeTypes = 3 316 | val nReleaseTypes = 6 317 | val nGrantTypes = 3 318 | 319 | val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) 320 | val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) 321 | val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) 322 | val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) 323 | 324 | def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) 325 | def grantTypesWithData = Vec(grantShared, grantExclusive) 326 | 327 | // Client states and functions 328 | val nClientStates = 3 329 | val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) 330 | 331 | def clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) 332 | def clientStatesWithWritePermission = Vec(clientExclusiveDirty) 333 | def clientStatesWithDirtyData = Vec(clientExclusiveDirty) 334 | 335 | def isValid(meta: ClientMetadata): Bool = meta.state =/= clientInvalid 336 | 337 | def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = 338 | Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) 339 | 340 | def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { 341 | val dirty = clientStatesWithDirtyData.contains(meta.state) 342 | MuxLookup(cmd, releaseCopyAck, Array( 343 | M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), 344 | M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), 345 | M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) 346 | } 347 | 348 | def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = 349 | MuxLookup(incoming.p_type, releaseInvalidateAck, Array( 350 | probeInvalidate -> getReleaseType(M_FLUSH, meta), 351 | probeDowngrade -> getReleaseType(M_PRODUCE, meta), 352 | probeCopy -> getReleaseType(M_CLEAN, meta))) 353 | 354 | def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = 355 | ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state))(meta.p) 356 | 357 | def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = 358 | ClientMetadata( 359 | MuxLookup(cmd, meta.state, Array( 360 | M_FLUSH -> clientInvalid, 361 | M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), 362 | clientShared, meta.state))))(meta.p) 363 | 364 | def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata) = 365 | ClientMetadata( 366 | Mux(incoming.isBuiltInType(), clientInvalid, 367 | MuxLookup(incoming.g_type, clientInvalid, Array( 368 | grantShared -> clientShared, 369 | grantExclusive -> clientExclusiveDirty, 370 | grantExclusiveAck -> clientExclusiveDirty))))(meta.p) 371 | 372 | def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = 373 | ClientMetadata( 374 | MuxLookup(incoming.p_type, meta.state, Array( 375 | probeInvalidate -> clientInvalid, 376 | probeDowngrade -> clientShared, 377 | probeCopy -> meta.state)))(meta.p) 378 | 379 | // Manager states and functions: 380 | val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing 381 | // only a single sharer (also would need 382 | // notification msg to track clean drops) 383 | // Also could avoid probes on outer WBs. 384 | 385 | def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = 386 | Mux(dir.none(meta.sharers), Bool(false), 387 | Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive 388 | Mux(a.isBuiltInType(), a.hasData(), a.a_type =/= acquireShared))) 389 | 390 | def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) 391 | 392 | def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = 393 | MuxLookup(cmd, probeCopy, Array( 394 | M_FLUSH -> probeInvalidate, 395 | M_PRODUCE -> probeDowngrade)) 396 | 397 | def getProbeType(a: HasAcquireType, meta: ManagerMetadata): UInt = 398 | Mux(a.isBuiltInType(), 399 | MuxLookup(a.a_type, probeCopy, Array( 400 | Acquire.getBlockType -> probeCopy, 401 | Acquire.putBlockType -> probeInvalidate, 402 | Acquire.getType -> probeCopy, 403 | Acquire.putType -> probeInvalidate, 404 | Acquire.getPrefetchType -> probeCopy, 405 | Acquire.putPrefetchType -> probeInvalidate, 406 | Acquire.putAtomicType -> probeInvalidate)), 407 | MuxLookup(a.a_type, probeCopy, Array( 408 | acquireShared -> probeDowngrade, 409 | acquireExclusive -> probeInvalidate))) 410 | 411 | def getGrantType(a: HasAcquireType, meta: ManagerMetadata): UInt = 412 | Mux(a.isBuiltInType(), Acquire.getBuiltInGrantType(a.a_type), 413 | Mux(a.a_type === acquireShared, 414 | Mux(!dir.none(meta.sharers), grantShared, grantExclusive), 415 | grantExclusive)) 416 | def getExclusiveGrantType(): UInt = grantExclusive 417 | 418 | def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { 419 | val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) 420 | MuxBundle(meta, Array( 421 | incoming.is(releaseInvalidateData) -> popped, 422 | incoming.is(releaseInvalidateAck) -> popped)) 423 | } 424 | } 425 | 426 | /** A protocol with four Client states. 427 | * Data is marked as dirty when written. 428 | * Multiple clients may share read permissions on a block at the same time. 429 | */ 430 | class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { 431 | // Message types 432 | val nAcquireTypes = 2 433 | val nProbeTypes = 3 434 | val nReleaseTypes = 6 435 | val nGrantTypes = 3 436 | 437 | val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) 438 | val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) 439 | val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) 440 | val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) 441 | 442 | def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) 443 | def grantTypesWithData = Vec(grantShared, grantExclusive) 444 | 445 | // Client states and functions 446 | val nClientStates = 4 447 | val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) 448 | 449 | def clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) 450 | def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) 451 | def clientStatesWithDirtyData = Vec(clientExclusiveDirty) 452 | 453 | def isValid(meta: ClientMetadata): Bool = meta.state =/= clientInvalid 454 | 455 | def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = 456 | Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) 457 | 458 | def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { 459 | val dirty = clientStatesWithDirtyData.contains(meta.state) 460 | MuxLookup(cmd, releaseCopyAck, Array( 461 | M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), 462 | M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), 463 | M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) 464 | } 465 | 466 | def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = 467 | MuxLookup(incoming.p_type, releaseInvalidateAck, Array( 468 | probeInvalidate -> getReleaseType(M_FLUSH, meta), 469 | probeDowngrade -> getReleaseType(M_PRODUCE, meta), 470 | probeCopy -> getReleaseType(M_CLEAN, meta))) 471 | 472 | def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = 473 | ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state))(meta.p) 474 | 475 | def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = 476 | ClientMetadata( 477 | MuxLookup(cmd, meta.state, Array( 478 | M_FLUSH -> clientInvalid, 479 | M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), 480 | clientShared, meta.state), 481 | M_CLEAN -> Mux(meta.state === clientExclusiveDirty, 482 | clientExclusiveClean, meta.state))))(meta.p) 483 | 484 | def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata) = 485 | ClientMetadata( 486 | Mux(incoming.isBuiltInType(), clientInvalid, 487 | MuxLookup(incoming.g_type, clientInvalid, Array( 488 | grantShared -> clientShared, 489 | grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), 490 | grantExclusiveAck -> clientExclusiveDirty))))(meta.p) 491 | 492 | def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = 493 | ClientMetadata( 494 | MuxLookup(incoming.p_type, meta.state, Array( 495 | probeInvalidate -> clientInvalid, 496 | probeDowngrade -> clientShared, 497 | probeCopy -> meta.state)))(meta.p) 498 | 499 | // Manager states and functions: 500 | val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing 501 | // only a single sharer (also would need 502 | // notification msg to track clean drops) 503 | // Also could avoid probes on outer WBs. 504 | 505 | def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = 506 | Mux(dir.none(meta.sharers), Bool(false), 507 | Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive 508 | Mux(a.isBuiltInType(), a.hasData(), a.a_type =/= acquireShared))) 509 | 510 | def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) 511 | 512 | def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = 513 | MuxLookup(cmd, probeCopy, Array( 514 | M_FLUSH -> probeInvalidate, 515 | M_PRODUCE -> probeDowngrade)) 516 | 517 | def getProbeType(a: HasAcquireType, meta: ManagerMetadata): UInt = 518 | Mux(a.isBuiltInType(), 519 | MuxLookup(a.a_type, probeCopy, Array( 520 | Acquire.getBlockType -> probeCopy, 521 | Acquire.putBlockType -> probeInvalidate, 522 | Acquire.getType -> probeCopy, 523 | Acquire.putType -> probeInvalidate, 524 | Acquire.getPrefetchType -> probeCopy, 525 | Acquire.putPrefetchType -> probeInvalidate, 526 | Acquire.putAtomicType -> probeInvalidate)), 527 | MuxLookup(a.a_type, probeCopy, Array( 528 | acquireShared -> probeDowngrade, 529 | acquireExclusive -> probeInvalidate))) 530 | 531 | def getGrantType(a: HasAcquireType, meta: ManagerMetadata): UInt = 532 | Mux(a.isBuiltInType(), Acquire.getBuiltInGrantType(a.a_type), 533 | Mux(a.a_type === acquireShared, 534 | Mux(!dir.none(meta.sharers), grantShared, grantExclusive), 535 | grantExclusive)) 536 | def getExclusiveGrantType(): UInt = grantExclusive 537 | 538 | def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { 539 | val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) 540 | MuxBundle(meta, Array( 541 | incoming.is(releaseInvalidateData) -> popped, 542 | incoming.is(releaseInvalidateAck) -> popped)) 543 | } 544 | } 545 | 546 | class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { 547 | // Message types 548 | val nAcquireTypes = 3 549 | val nProbeTypes = 4 550 | val nReleaseTypes = 10 551 | val nGrantTypes = 4 552 | 553 | val acquireShared :: acquireExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) 554 | val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) 555 | val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) 556 | val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) 557 | 558 | def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) 559 | def grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) 560 | 561 | // Client states and functions 562 | val nClientStates = 7 563 | val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) 564 | 565 | def clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) 566 | def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) 567 | def clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) 568 | 569 | def isValid (meta: ClientMetadata): Bool = meta.state =/= clientInvalid 570 | 571 | def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = 572 | Mux(isWriteIntent(cmd), 573 | Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), 574 | acquireShared) 575 | 576 | def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { 577 | val dirty = clientStatesWithDirtyData.contains(meta.state) 578 | MuxLookup(cmd, releaseCopyAck, Array( 579 | M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), 580 | M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), 581 | M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) 582 | } 583 | 584 | def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = { 585 | val dirty = clientStatesWithDirtyData.contains(meta.state) 586 | val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( 587 | probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(meta.state), 588 | releaseInvalidateDataMigratory, releaseInvalidateData), 589 | probeDowngrade -> Mux(meta.state === clientMigratoryDirty, 590 | releaseDowngradeDataMigratory, releaseDowngradeData), 591 | probeCopy -> releaseCopyData)) 592 | val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( 593 | probeInvalidate -> Mux(clientExclusiveClean === meta.state, 594 | releaseInvalidateAckMigratory, releaseInvalidateAck), 595 | probeInvalidateOthers -> Mux(clientSharedByTwo === meta.state, 596 | releaseInvalidateAckMigratory, releaseInvalidateAck), 597 | probeDowngrade -> Mux(meta.state =/= clientInvalid, 598 | releaseDowngradeAckHasCopy, releaseDowngradeAck), 599 | probeCopy -> releaseCopyAck)) 600 | Mux(dirty, with_data, without_data) 601 | } 602 | 603 | def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = 604 | ClientMetadata( 605 | Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( 606 | clientExclusiveClean -> clientExclusiveDirty, 607 | clientMigratoryClean -> clientMigratoryDirty)), 608 | meta.state))(meta.p) 609 | 610 | def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = 611 | ClientMetadata( 612 | MuxLookup(cmd, meta.state, Array( 613 | M_FLUSH -> clientInvalid, 614 | M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), 615 | clientShared, meta.state), 616 | M_CLEAN -> MuxLookup(meta.state, meta.state, Array( 617 | clientExclusiveDirty -> clientExclusiveClean, 618 | clientMigratoryDirty -> clientMigratoryClean)))))(meta.p) 619 | 620 | def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata) = 621 | ClientMetadata( 622 | Mux(incoming.isBuiltInType(), clientInvalid, 623 | MuxLookup(incoming.g_type, clientInvalid, Array( 624 | grantShared -> clientShared, 625 | grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), 626 | grantExclusiveAck -> clientExclusiveDirty, 627 | grantReadMigratory -> Mux(isWrite(cmd), 628 | clientMigratoryDirty, clientMigratoryClean)))))(meta.p) 629 | 630 | def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = 631 | ClientMetadata( 632 | MuxLookup(incoming.p_type, meta.state, Array( 633 | probeInvalidate -> clientInvalid, 634 | probeInvalidateOthers -> clientInvalid, 635 | probeCopy -> meta.state, 636 | probeDowngrade -> MuxLookup(meta.state, clientShared, Array( 637 | clientExclusiveClean -> clientSharedByTwo, 638 | clientExclusiveDirty -> clientSharedByTwo, 639 | clientSharedByTwo -> clientShared, 640 | clientMigratoryClean -> clientSharedByTwo, 641 | clientMigratoryDirty -> clientInvalid)))))(meta.p) 642 | 643 | // Manager states and functions: 644 | val nManagerStates = 0 // TODO: we could add some states to reduce the number of message types 645 | 646 | def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = 647 | Mux(dir.none(meta.sharers), Bool(false), 648 | Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive 649 | Mux(a.isBuiltInType(), a.hasData(), a.a_type =/= acquireShared))) 650 | 651 | def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) 652 | 653 | def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = 654 | MuxLookup(cmd, probeCopy, Array( 655 | M_FLUSH -> probeInvalidate, 656 | M_PRODUCE -> probeDowngrade)) 657 | 658 | def getProbeType(a: HasAcquireType, meta: ManagerMetadata): UInt = 659 | Mux(a.isBuiltInType(), 660 | MuxLookup(a.a_type, probeCopy, Array( 661 | Acquire.getBlockType -> probeCopy, 662 | Acquire.putBlockType -> probeInvalidate, 663 | Acquire.getType -> probeCopy, 664 | Acquire.putType -> probeInvalidate, 665 | Acquire.getPrefetchType -> probeCopy, 666 | Acquire.putPrefetchType -> probeInvalidate, 667 | Acquire.putAtomicType -> probeInvalidate)), 668 | MuxLookup(a.a_type, probeCopy, Array( 669 | acquireShared -> probeDowngrade, 670 | acquireExclusive -> probeInvalidate, 671 | acquireInvalidateOthers -> probeInvalidateOthers))) 672 | 673 | def getGrantType(a: HasAcquireType, meta: ManagerMetadata): UInt = 674 | Mux(a.isBuiltInType(), Acquire.getBuiltInGrantType(a.a_type), 675 | MuxLookup(a.a_type, grantShared, Array( 676 | acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), 677 | acquireExclusive -> grantExclusive, 678 | acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? 679 | def getExclusiveGrantType(): UInt = grantExclusive 680 | 681 | def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { 682 | val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) 683 | MuxBundle(meta, Array( 684 | incoming.is(releaseInvalidateData) -> popped, 685 | incoming.is(releaseInvalidateAck) -> popped, 686 | incoming.is(releaseInvalidateDataMigratory) -> popped, 687 | incoming.is(releaseInvalidateAckMigratory) -> popped)) 688 | } 689 | } 690 | -------------------------------------------------------------------------------- /src/main/scala/consts.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | package constants 5 | 6 | import Chisel._ 7 | 8 | object MemoryOpConstants extends MemoryOpConstants 9 | trait MemoryOpConstants { 10 | val MT_SZ = 4 11 | val MT_X = BitPat("b????") 12 | val MT_B = UInt("b0000") 13 | val MT_H = UInt("b0001") 14 | val MT_W = UInt("b0010") 15 | val MT_D = UInt("b0011") 16 | val MT_BU = UInt("b0100") 17 | val MT_HU = UInt("b0101") 18 | val MT_WU = UInt("b0110") 19 | val MT_Q = UInt("b0111") 20 | val MT_T = UInt("b1111") // tag 21 | 22 | val NUM_XA_OPS = 9 23 | val M_SZ = 5 24 | val M_X = BitPat("b?????"); 25 | val M_XRD = UInt("b00000"); // int load 26 | val M_XWR = UInt("b00001"); // int store 27 | val M_PFR = UInt("b00010"); // prefetch with intent to read 28 | val M_PFW = UInt("b00011"); // prefetch with intent to write 29 | val M_XA_SWAP = UInt("b00100"); 30 | val M_NOP = UInt("b00101"); 31 | val M_XLR = UInt("b00110"); 32 | val M_XSC = UInt("b00111"); 33 | val M_XA_ADD = UInt("b01000"); 34 | val M_XA_XOR = UInt("b01001"); 35 | val M_XA_OR = UInt("b01010"); 36 | val M_XA_AND = UInt("b01011"); 37 | val M_XA_MIN = UInt("b01100"); 38 | val M_XA_MAX = UInt("b01101"); 39 | val M_XA_MINU = UInt("b01110"); 40 | val M_XA_MAXU = UInt("b01111"); 41 | val M_FLUSH = UInt("b10000") // write back dirty data and cede R/W permissions 42 | val M_PRODUCE = UInt("b10001") // write back dirty data and cede W permissions 43 | val M_CLEAN = UInt("b10011") // write back dirty data and retain R/W permissions 44 | 45 | def isAMO(cmd: UInt) = cmd(3) || cmd === M_XA_SWAP 46 | def isPrefetch(cmd: UInt) = cmd === M_PFR || cmd === M_PFW 47 | def isRead(cmd: UInt) = cmd === M_XRD || cmd === M_XLR || cmd === M_XSC || isAMO(cmd) 48 | def isWrite(cmd: UInt) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) 49 | def isWriteIntent(cmd: UInt) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR 50 | } 51 | -------------------------------------------------------------------------------- /src/main/scala/directory.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | import Chisel._ 5 | 6 | // This class encapsulates transformations on different directory information 7 | // storage formats 8 | abstract class DirectoryRepresentation(val width: Int) { 9 | def pop(prev: UInt, id: UInt): UInt 10 | def push(prev: UInt, id: UInt): UInt 11 | def flush: UInt 12 | def none(s: UInt): Bool 13 | def one(s: UInt): Bool 14 | def count(s: UInt): UInt 15 | def next(s: UInt): UInt 16 | def full(s: UInt): UInt 17 | } 18 | 19 | abstract trait HasDirectoryRepresentation { 20 | val dir: DirectoryRepresentation 21 | } 22 | 23 | class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) { 24 | def pop(prev: UInt, id: UInt) = UInt(0) 25 | def push(prev: UInt, id: UInt) = UInt(0) 26 | def flush = UInt(0) 27 | def none(s: UInt) = Bool(false) 28 | def one(s: UInt) = Bool(false) 29 | def count(s: UInt) = UInt(nClients) 30 | def next(s: UInt) = UInt(0) 31 | def full(s: UInt) = SInt(-1, width = nClients).toUInt 32 | } 33 | 34 | class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { 35 | def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) 36 | def push(prev: UInt, id: UInt) = prev | UIntToOH(id) 37 | def flush = UInt(0, width = width) 38 | def none(s: UInt) = s === UInt(0) 39 | def one(s: UInt) = PopCount(s) === UInt(1) 40 | def count(s: UInt) = PopCount(s) 41 | def next(s: UInt) = PriorityEncoder(s) 42 | def full(s: UInt) = s 43 | } 44 | -------------------------------------------------------------------------------- /src/main/scala/dma.scala: -------------------------------------------------------------------------------- 1 | package uncore 2 | 3 | import Chisel._ 4 | import cde.{Parameters, Field} 5 | import junctions._ 6 | import junctions.NastiConstants._ 7 | 8 | case object NDmaTransactors extends Field[Int] 9 | case object NDmaXacts extends Field[Int] 10 | case object NDmaClients extends Field[Int] 11 | 12 | trait HasDmaParameters { 13 | implicit val p: Parameters 14 | val nDmaTransactors = p(NDmaTransactors) 15 | val nDmaXacts = p(NDmaXacts) 16 | val nDmaClients = p(NDmaClients) 17 | val dmaXactIdBits = log2Up(nDmaXacts) 18 | val dmaClientIdBits = log2Up(nDmaClients) 19 | val addrBits = p(PAddrBits) 20 | val dmaStatusBits = 2 21 | val dmaWordSizeBits = 2 22 | } 23 | 24 | abstract class DmaModule(implicit val p: Parameters) extends Module with HasDmaParameters 25 | abstract class DmaBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) with HasDmaParameters 26 | 27 | class DmaRequest(implicit p: Parameters) extends DmaBundle()(p) { 28 | val xact_id = UInt(width = dmaXactIdBits) 29 | val client_id = UInt(width = dmaClientIdBits) 30 | val cmd = UInt(width = DmaRequest.DMA_CMD_SZ) 31 | val source = UInt(width = addrBits) 32 | val dest = UInt(width = addrBits) 33 | val length = UInt(width = addrBits) 34 | val size = UInt(width = dmaWordSizeBits) 35 | } 36 | 37 | class DmaResponse(implicit p: Parameters) extends DmaBundle()(p) { 38 | val xact_id = UInt(width = dmaXactIdBits) 39 | val client_id = UInt(width = dmaClientIdBits) 40 | val status = UInt(width = dmaStatusBits) 41 | } 42 | 43 | object DmaRequest { 44 | val DMA_CMD_SZ = 3 45 | 46 | val DMA_CMD_COPY = UInt("b000") 47 | val DMA_CMD_PFR = UInt("b010") 48 | val DMA_CMD_PFW = UInt("b011") 49 | val DMA_CMD_SIN = UInt("b100") 50 | val DMA_CMD_SOUT = UInt("b101") 51 | 52 | def apply(xact_id: UInt = UInt(0), 53 | client_id: UInt, 54 | cmd: UInt, 55 | source: UInt, 56 | dest: UInt, 57 | length: UInt, 58 | size: UInt = UInt(0))(implicit p: Parameters): DmaRequest = { 59 | val req = Wire(new DmaRequest) 60 | req.xact_id := xact_id 61 | req.client_id := client_id 62 | req.cmd := cmd 63 | req.source := source 64 | req.dest := dest 65 | req.length := length 66 | req.size := size 67 | req 68 | } 69 | } 70 | import DmaRequest._ 71 | 72 | class DmaIO(implicit p: Parameters) extends DmaBundle()(p) { 73 | val req = Decoupled(new DmaRequest) 74 | val resp = Decoupled(new DmaResponse).flip 75 | } 76 | 77 | class DmaTrackerIO(implicit p: Parameters) extends DmaBundle()(p) { 78 | val dma = (new DmaIO).flip 79 | val mem = new ClientUncachedTileLinkIO 80 | val mmio = new NastiIO 81 | } 82 | 83 | class DmaManager(outstandingCSR: Int)(implicit p: Parameters) 84 | extends DmaModule()(p) 85 | with HasNastiParameters 86 | with HasAddrMapParameters 87 | with HasSCRParameters { 88 | 89 | val io = new Bundle { 90 | val ctrl = (new NastiIO).flip 91 | val mmio = new NastiIO 92 | val dma = new DmaIO 93 | } 94 | 95 | private val wordBits = 1 << log2Up(addrBits) 96 | private val wordBytes = wordBits / 8 97 | private val wordOff = log2Up(wordBytes) 98 | private val wordMSB = wordOff + 2 99 | 100 | val s_idle :: s_wdata :: s_dma_req :: s_wresp :: Nil = Enum(Bits(), 4) 101 | val state = Reg(init = s_idle) 102 | 103 | val nCtrlWords = (addrBits * 4) / nastiXDataBits 104 | val ctrl_regs = Reg(Vec(nCtrlWords, UInt(width = nastiXDataBits))) 105 | val ctrl_idx = Reg(UInt(width = log2Up(nCtrlWords))) 106 | val ctrl_done = Reg(Bool()) 107 | val ctrl_blob = ctrl_regs.toBits 108 | val ctrl_id = Reg(UInt(width = nastiXIdBits)) 109 | 110 | val sizeOffset = 3 * addrBits 111 | val cmdOffset = sizeOffset + dmaWordSizeBits 112 | 113 | val dma_req = new DmaRequest().fromBits(ctrl_blob) 114 | val dma_busy = Reg(init = UInt(0, nDmaXacts)) 115 | val dma_xact_id = PriorityEncoder(~dma_busy) 116 | 117 | when (io.ctrl.aw.fire()) { 118 | ctrl_id := io.ctrl.aw.bits.id 119 | ctrl_idx := UInt(0) 120 | ctrl_done := Bool(false) 121 | state := s_wdata 122 | } 123 | 124 | when (io.ctrl.w.fire()) { 125 | when (!ctrl_done) { 126 | ctrl_regs(ctrl_idx) := io.ctrl.w.bits.data 127 | ctrl_idx := ctrl_idx + UInt(1) 128 | } 129 | when (ctrl_idx === UInt(nCtrlWords - 1)) { ctrl_done := Bool(true) } 130 | when (io.ctrl.w.bits.last) { state := s_dma_req } 131 | } 132 | 133 | dma_busy := (dma_busy | 134 | Mux(io.dma.req.fire(), UIntToOH(dma_xact_id), UInt(0))) & 135 | ~Mux(io.dma.resp.fire(), UIntToOH(io.dma.resp.bits.xact_id), UInt(0)) 136 | 137 | when (io.dma.req.fire()) { state := s_wresp } 138 | when (io.ctrl.b.fire()) { state := s_idle } 139 | 140 | io.ctrl.ar.ready := Bool(false) 141 | io.ctrl.aw.ready := (state === s_idle) 142 | io.ctrl.w.ready := (state === s_wdata) 143 | 144 | io.ctrl.r.valid := Bool(false) 145 | io.ctrl.b.valid := (state === s_wresp) 146 | io.ctrl.b.bits := NastiWriteResponseChannel(id = ctrl_id) 147 | 148 | io.dma.req.valid := (state === s_dma_req) && !dma_busy.andR 149 | io.dma.req.bits := dma_req 150 | io.dma.req.bits.xact_id := dma_xact_id 151 | 152 | val resp_waddr_pending = Reg(init = Bool(false)) 153 | val resp_wdata_pending = Reg(init = Bool(false)) 154 | val resp_wresp_pending = Reg(init = Bool(false)) 155 | val resp_pending = resp_waddr_pending || resp_wdata_pending || resp_wresp_pending 156 | 157 | val resp_client_id = Reg(UInt(width = dmaClientIdBits)) 158 | val resp_status = Reg(UInt(width = dmaStatusBits)) 159 | 160 | io.dma.resp.ready := !resp_pending 161 | 162 | when (io.dma.resp.fire()) { 163 | resp_client_id := io.dma.resp.bits.client_id 164 | resp_status := io.dma.resp.bits.status 165 | resp_waddr_pending := Bool(true) 166 | resp_wdata_pending := Bool(true) 167 | resp_wresp_pending := Bool(true) 168 | } 169 | 170 | val addrTable = Vec.tabulate(nDmaClients) { i => 171 | UInt(addrMap(s"conf:csr$i").start + outstandingCSR * csrDataBytes) 172 | } 173 | 174 | io.mmio.ar.valid := Bool(false) 175 | io.mmio.aw.valid := resp_waddr_pending 176 | io.mmio.aw.bits := NastiWriteAddressChannel( 177 | id = UInt(0), 178 | addr = addrTable(resp_client_id), 179 | size = UInt(log2Up(csrDataBytes))) 180 | io.mmio.w.valid := resp_wdata_pending 181 | io.mmio.w.bits := NastiWriteDataChannel(data = resp_status) 182 | io.mmio.b.ready := resp_wresp_pending 183 | io.mmio.r.ready := Bool(false) 184 | 185 | when (io.mmio.aw.fire()) { resp_waddr_pending := Bool(false) } 186 | when (io.mmio.w.fire()) { resp_wdata_pending := Bool(false) } 187 | when (io.mmio.b.fire()) { resp_wresp_pending := Bool(false) } 188 | } 189 | 190 | class DmaEngine(outstandingCSR: Int)(implicit p: Parameters) extends DmaModule()(p) { 191 | val io = new Bundle { 192 | val ctrl = (new NastiIO).flip 193 | val mem = new ClientUncachedTileLinkIO 194 | val mmio = new NastiIO 195 | } 196 | 197 | val manager = Module(new DmaManager(outstandingCSR)) 198 | val trackers = Module(new DmaTrackerFile) 199 | 200 | manager.io.ctrl <> io.ctrl 201 | trackers.io.dma <> manager.io.dma 202 | 203 | val innerIOs = trackers.io.mem 204 | val outerIOs = trackers.io.mmio :+ manager.io.mmio 205 | 206 | val innerArb = Module(new ClientUncachedTileLinkIOArbiter(innerIOs.size)) 207 | innerArb.io.in <> innerIOs 208 | io.mem <> innerArb.io.out 209 | 210 | val outerArb = Module(new NastiArbiter(outerIOs.size)) 211 | outerArb.io.master <> outerIOs 212 | io.mmio <> outerArb.io.slave 213 | 214 | assert(!io.mmio.b.valid || io.mmio.b.bits.resp === UInt(0), 215 | "DmaEngine: NASTI write response error") 216 | 217 | assert(!io.mmio.r.valid || io.mmio.r.bits.resp === UInt(0), 218 | "DmaEngine: NASTI read response error") 219 | } 220 | 221 | class DmaTrackerFile(implicit p: Parameters) extends DmaModule()(p) { 222 | val io = new Bundle { 223 | val dma = (new DmaIO).flip 224 | val mem = Vec(nDmaTransactors, new ClientUncachedTileLinkIO) 225 | val mmio = Vec(nDmaTransactors, new NastiIO) 226 | } 227 | 228 | val trackers = List.fill(nDmaTransactors) { Module(new DmaTracker) } 229 | val reqReadys = Vec(trackers.map(_.io.dma.req.ready)).toBits 230 | 231 | io.mem <> trackers.map(_.io.mem) 232 | io.mmio <> trackers.map(_.io.mmio) 233 | 234 | if (nDmaTransactors > 1) { 235 | val resp_arb = Module(new RRArbiter(new DmaResponse, nDmaTransactors)) 236 | resp_arb.io.in <> trackers.map(_.io.dma.resp) 237 | io.dma.resp <> resp_arb.io.out 238 | 239 | val selection = PriorityEncoder(reqReadys) 240 | trackers.zipWithIndex.foreach { case (tracker, i) => 241 | tracker.io.dma.req.valid := io.dma.req.valid && selection === UInt(i) 242 | tracker.io.dma.req.bits := io.dma.req.bits 243 | } 244 | io.dma.req.ready := reqReadys.orR 245 | } else { 246 | io.dma <> trackers.head.io.dma 247 | } 248 | } 249 | 250 | class DmaTracker(implicit p: Parameters) extends DmaModule()(p) 251 | with HasTileLinkParameters with HasNastiParameters { 252 | val io = new DmaTrackerIO 253 | 254 | private val blockOffset = tlBeatAddrBits + tlByteAddrBits 255 | private val blockBytes = tlDataBeats * tlDataBytes 256 | 257 | val data_buffer = Reg(Vec(2 * tlDataBeats, Bits(width = tlDataBits))) 258 | val get_inflight = Reg(UInt(2 * tlDataBeats)) 259 | val put_inflight = Reg(Bool()) 260 | val put_half = Reg(UInt(width = 1)) 261 | val get_half = Reg(UInt(width = 1)) 262 | val prefetch_put = Reg(Bool()) 263 | val get_done = !get_inflight.orR 264 | 265 | val src_block = Reg(UInt(width = tlBlockAddrBits)) 266 | val dst_block = Reg(UInt(width = tlBlockAddrBits)) 267 | val offset = Reg(UInt(width = blockOffset)) 268 | val alignment = Reg(UInt(width = blockOffset)) 269 | val shift_dir = Reg(Bool()) 270 | 271 | val bytes_left = Reg(UInt(width = addrBits)) 272 | val streaming = Reg(Bool()) 273 | val stream_addr = Reg(UInt(width = nastiXAddrBits)) 274 | val stream_len = Reg(UInt(width = nastiXLenBits)) 275 | val stream_size = Reg(UInt(width = nastiXSizeBits)) 276 | val stream_idx = Reg(UInt(width = blockOffset)) 277 | val stream_bytesel = MuxLookup(stream_size, UInt("b11111111"), Seq( 278 | UInt("b00") -> UInt("b00000001"), 279 | UInt("b01") -> UInt("b00000011"), 280 | UInt("b10") -> UInt("b00001111"))) 281 | val stream_mask = FillInterleaved(8, stream_bytesel) 282 | val stream_last = Reg(Bool()) 283 | 284 | val stream_word_bytes = UInt(1) << stream_size 285 | val stream_beat_idx = stream_idx(blockOffset - 1, tlByteAddrBits) 286 | val stream_byte_idx = stream_idx(tlByteAddrBits - 1, 0) 287 | val stream_bitshift = Cat(stream_byte_idx, UInt(0, 3)) 288 | val stream_in_beat = 289 | (((io.mmio.r.bits.data & stream_mask) << stream_bitshift)) | 290 | (data_buffer(stream_beat_idx) & ~(stream_mask << stream_bitshift)) 291 | val stream_out_word = data_buffer(stream_beat_idx) >> stream_bitshift 292 | val stream_out_last = bytes_left === stream_word_bytes 293 | 294 | val acq = io.mem.acquire.bits 295 | val gnt = io.mem.grant.bits 296 | 297 | val (s_idle :: s_get :: s_put :: s_prefetch :: 298 | s_stream_read_req :: s_stream_read_resp :: 299 | s_stream_write_req :: s_stream_write_data :: s_stream_write_resp :: 300 | s_wait :: s_resp :: Nil) = Enum(Bits(), 11) 301 | val state = Reg(init = s_idle) 302 | 303 | val (put_beat, put_done) = Counter( 304 | io.mem.acquire.fire() && acq.hasData(), tlDataBeats) 305 | 306 | val put_mask = Vec.tabulate(tlDataBytes) { i => 307 | val byte_index = Cat(put_beat, UInt(i, tlByteAddrBits)) 308 | byte_index >= offset && byte_index < bytes_left 309 | }.toBits 310 | 311 | val prefetch_sent = io.mem.acquire.fire() && io.mem.acquire.bits.isPrefetch() 312 | val prefetch_busy = Reg(init = UInt(0, tlMaxClientXacts)) 313 | val (prefetch_id, _) = Counter(prefetch_sent, tlMaxClientXacts) 314 | 315 | val base_index = Cat(put_half, put_beat) 316 | val put_data = Wire(init = Bits(0, tlDataBits)) 317 | val beat_align = alignment(blockOffset - 1, tlByteAddrBits) 318 | val bit_align = Cat(alignment(tlByteAddrBits - 1, 0), UInt(0, 3)) 319 | val rev_align = UInt(tlDataBits) - bit_align 320 | 321 | def getBit(value: UInt, sel: UInt): Bool = 322 | (value >> sel)(0) 323 | 324 | when (alignment === UInt(0)) { 325 | put_data := data_buffer.read(base_index) 326 | } .elsewhen (shift_dir) { 327 | val shift_index = base_index - beat_align 328 | when (bit_align === UInt(0)) { 329 | put_data := data_buffer.read(shift_index) 330 | } .otherwise { 331 | val upper_bits = data_buffer.read(shift_index) 332 | val lower_bits = data_buffer.read(shift_index - UInt(1)) 333 | val upper_shifted = upper_bits << bit_align 334 | val lower_shifted = lower_bits >> rev_align 335 | put_data := upper_shifted | lower_shifted 336 | } 337 | } .otherwise { 338 | val shift_index = base_index + beat_align 339 | when (bit_align === UInt(0)) { 340 | put_data := data_buffer.read(shift_index) 341 | } .otherwise { 342 | val upper_bits = data_buffer.read(shift_index + UInt(1)) 343 | val lower_bits = data_buffer.read(shift_index) 344 | val upper_shifted = upper_bits << rev_align 345 | val lower_shifted = lower_bits >> bit_align 346 | put_data := upper_shifted | lower_shifted 347 | } 348 | } 349 | 350 | val put_acquire = PutBlock( 351 | client_xact_id = UInt(2), 352 | addr_block = dst_block, 353 | addr_beat = put_beat, 354 | data = put_data, 355 | wmask = put_mask) 356 | 357 | val get_acquire = GetBlock( 358 | client_xact_id = get_half, 359 | addr_block = src_block, 360 | alloc = Bool(false)) 361 | 362 | val prefetch_acquire = Mux(prefetch_put, 363 | PutPrefetch(client_xact_id = prefetch_id, addr_block = dst_block), 364 | GetPrefetch(client_xact_id = prefetch_id, addr_block = dst_block)) 365 | 366 | val resp_xact_id = Reg(UInt(width = dmaXactIdBits)) 367 | val resp_client_id = Reg(UInt(width = dmaClientIdBits)) 368 | 369 | io.mem.acquire.valid := (state === s_get) || 370 | (state === s_put && get_done) || 371 | (state === s_prefetch && !prefetch_busy(prefetch_id)) 372 | io.mem.acquire.bits := MuxBundle( 373 | state, prefetch_acquire, Seq( 374 | s_get -> get_acquire, 375 | s_put -> put_acquire)) 376 | io.mem.grant.ready := Bool(true) 377 | io.dma.req.ready := state === s_idle 378 | io.dma.resp.valid := state === s_resp 379 | io.dma.resp.bits.xact_id := resp_xact_id 380 | io.dma.resp.bits.client_id := resp_client_id 381 | io.dma.resp.bits.status := UInt(0) 382 | io.mmio.ar.valid := (state === s_stream_read_req) 383 | io.mmio.ar.bits := NastiReadAddressChannel( 384 | id = UInt(0), 385 | addr = stream_addr, 386 | size = stream_size, 387 | len = stream_len, 388 | burst = BURST_FIXED) 389 | io.mmio.r.ready := (state === s_stream_read_resp) 390 | 391 | io.mmio.aw.valid := (state === s_stream_write_req) 392 | io.mmio.aw.bits := NastiWriteAddressChannel( 393 | id = UInt(0), 394 | addr = stream_addr, 395 | size = stream_size, 396 | len = stream_len, 397 | burst = BURST_FIXED) 398 | io.mmio.w.valid := (state === s_stream_write_data) && get_done 399 | io.mmio.w.bits := NastiWriteDataChannel( 400 | data = stream_out_word, 401 | last = stream_out_last) 402 | io.mmio.b.ready := (state === s_stream_write_resp) 403 | 404 | when (io.dma.req.fire()) { 405 | val src_off = io.dma.req.bits.source(blockOffset - 1, 0) 406 | val dst_off = io.dma.req.bits.dest(blockOffset - 1, 0) 407 | val direction = src_off < dst_off 408 | 409 | resp_xact_id := io.dma.req.bits.xact_id 410 | resp_client_id := io.dma.req.bits.client_id 411 | src_block := io.dma.req.bits.source(addrBits - 1, blockOffset) 412 | dst_block := io.dma.req.bits.dest(addrBits - 1, blockOffset) 413 | alignment := Mux(direction, dst_off - src_off, src_off - dst_off) 414 | shift_dir := direction 415 | offset := dst_off 416 | bytes_left := io.dma.req.bits.length + dst_off 417 | get_inflight := UInt(0) 418 | put_inflight := Bool(false) 419 | get_half := UInt(0) 420 | put_half := UInt(0) 421 | streaming := Bool(false) 422 | stream_len := (io.dma.req.bits.length >> io.dma.req.bits.size) - UInt(1) 423 | stream_size := io.dma.req.bits.size 424 | stream_last := Bool(false) 425 | 426 | when (io.dma.req.bits.cmd === DMA_CMD_COPY) { 427 | state := s_get 428 | } .elsewhen (io.dma.req.bits.cmd(2, 1) === UInt("b01")) { 429 | prefetch_put := io.dma.req.bits.cmd(0) 430 | state := s_prefetch 431 | } .elsewhen (io.dma.req.bits.cmd === DMA_CMD_SIN) { 432 | stream_addr := io.dma.req.bits.source 433 | stream_idx := dst_off 434 | streaming := Bool(true) 435 | alignment := UInt(0) 436 | state := s_stream_read_req 437 | } .elsewhen (io.dma.req.bits.cmd === DMA_CMD_SOUT) { 438 | stream_addr := io.dma.req.bits.dest 439 | stream_idx := src_off 440 | streaming := Bool(true) 441 | bytes_left := io.dma.req.bits.length 442 | state := s_stream_write_req 443 | } 444 | } 445 | 446 | when (io.mmio.ar.fire()) { state := s_stream_read_resp } 447 | 448 | when (io.mmio.r.fire()) { 449 | data_buffer(stream_beat_idx) := stream_in_beat 450 | stream_idx := stream_idx + stream_word_bytes 451 | val block_finished = stream_idx === UInt(blockBytes) - stream_word_bytes 452 | when (block_finished || io.mmio.r.bits.last) { state := s_put } 453 | } 454 | 455 | when (io.mmio.aw.fire()) { state := s_get } 456 | 457 | when (io.mmio.w.fire()) { 458 | stream_idx := stream_idx + stream_word_bytes 459 | bytes_left := bytes_left - stream_word_bytes 460 | val block_finished = stream_idx === UInt(blockBytes) - stream_word_bytes 461 | when (stream_out_last) { 462 | state := s_stream_write_resp 463 | } .elsewhen (block_finished) { 464 | state := s_get 465 | } 466 | } 467 | 468 | when (io.mmio.b.fire()) { state := s_resp } 469 | 470 | when (state === s_get && io.mem.acquire.ready) { 471 | get_inflight := get_inflight | FillInterleaved(tlDataBeats, UIntToOH(get_half)) 472 | src_block := src_block + UInt(1) 473 | when (streaming) { 474 | state := s_stream_write_data 475 | } .otherwise { 476 | val bytes_in_buffer = UInt(blockBytes) - alignment 477 | val extra_read = alignment > UInt(0) && !shift_dir && // dst_off < src_off 478 | get_half === UInt(0) && // this is the first block 479 | bytes_in_buffer < bytes_left // there is still more data left to fetch 480 | get_half := get_half + UInt(1) 481 | when (!extra_read) { state := s_put } 482 | } 483 | } 484 | 485 | when (prefetch_sent) { 486 | prefetch_busy := prefetch_busy | UIntToOH(prefetch_id) 487 | when (bytes_left < UInt(blockBytes)) { 488 | bytes_left := UInt(0) 489 | state := s_resp 490 | } .otherwise { 491 | bytes_left := bytes_left - UInt(blockBytes) 492 | dst_block := dst_block + UInt(1) 493 | } 494 | } 495 | 496 | when (io.mem.grant.fire()) { 497 | when (gnt.g_type === Grant.prefetchAckType) { 498 | prefetch_busy := prefetch_busy & ~UIntToOH(gnt.client_xact_id) 499 | } .elsewhen (gnt.hasData()) { 500 | val write_half = gnt.client_xact_id(0) 501 | val write_idx = Cat(write_half, gnt.addr_beat) 502 | get_inflight := get_inflight & ~UIntToOH(write_idx) 503 | data_buffer.write(write_idx, gnt.data) 504 | } .otherwise { 505 | put_inflight := Bool(false) 506 | } 507 | } 508 | 509 | when (put_done) { // state === s_put 510 | when (!streaming) { 511 | put_half := put_half + UInt(1) 512 | } 513 | offset := UInt(0) 514 | stream_idx := UInt(0) 515 | when (bytes_left < UInt(blockBytes)) { 516 | bytes_left := UInt(0) 517 | } .otherwise { 518 | bytes_left := bytes_left - UInt(blockBytes) 519 | } 520 | put_inflight := Bool(true) 521 | dst_block := dst_block + UInt(1) 522 | state := s_wait 523 | } 524 | 525 | when (state === s_wait && get_done && !put_inflight) { 526 | state := MuxCase(s_get, Seq( 527 | (bytes_left === UInt(0)) -> s_resp, 528 | streaming -> s_stream_read_resp)) 529 | } 530 | 531 | when (io.dma.resp.fire()) { state := s_idle } 532 | } 533 | -------------------------------------------------------------------------------- /src/main/scala/ecc.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | 5 | import Chisel._ 6 | 7 | abstract class Decoding 8 | { 9 | def uncorrected: UInt 10 | def corrected: UInt 11 | def correctable: Bool 12 | def uncorrectable: Bool 13 | def error = correctable || uncorrectable 14 | } 15 | 16 | abstract class Code 17 | { 18 | def width(w0: Int): Int 19 | def encode(x: UInt): UInt 20 | def decode(x: UInt): Decoding 21 | } 22 | 23 | class IdentityCode extends Code 24 | { 25 | def width(w0: Int) = w0 26 | def encode(x: UInt) = x 27 | def decode(y: UInt) = new Decoding { 28 | def uncorrected = y 29 | def corrected = y 30 | def correctable = Bool(false) 31 | def uncorrectable = Bool(false) 32 | } 33 | } 34 | 35 | class ParityCode extends Code 36 | { 37 | def width(w0: Int) = w0+1 38 | def encode(x: UInt) = Cat(x.xorR, x) 39 | def decode(y: UInt) = new Decoding { 40 | def uncorrected = y(y.getWidth-2,0) 41 | def corrected = uncorrected 42 | def correctable = Bool(false) 43 | def uncorrectable = y.xorR 44 | } 45 | } 46 | 47 | class SECCode extends Code 48 | { 49 | def width(k: Int) = { 50 | val m = new Unsigned(k).log2 + 1 51 | k + m + (if((1 << m) < m+k+1) 1 else 0) 52 | } 53 | def encode(x: UInt) = { 54 | val k = x.getWidth 55 | require(k > 0) 56 | val n = width(k) 57 | 58 | val y = for (i <- 1 to n) yield { 59 | if (isPow2(i)) { 60 | val r = for (j <- 1 to n; if j != i && (j & i) != 0) 61 | yield x(mapping(j)) 62 | r reduce (_^_) 63 | } else 64 | x(mapping(i)) 65 | } 66 | Vec(y).toBits 67 | } 68 | def decode(y: UInt) = new Decoding { 69 | val n = y.getWidth 70 | require(n > 0 && !isPow2(n)) 71 | 72 | val p2 = for (i <- 0 until log2Up(n)) yield 1 << i 73 | val syndrome = p2 map { i => 74 | val r = for (j <- 1 to n; if (j & i) != 0) 75 | yield y(j-1) 76 | r reduce (_^_) 77 | } 78 | val s = Vec(syndrome).toBits 79 | 80 | private def swizzle(z: UInt) = Vec((1 to n).filter(i => !isPow2(i)).map(i => z(i-1))).toBits 81 | def uncorrected = swizzle(y) 82 | def corrected = swizzle(((y.toUInt << 1) ^ UIntToOH(s)) >> 1) 83 | def correctable = s.orR 84 | def uncorrectable = Bool(false) 85 | } 86 | private def mapping(i: Int) = i-1-log2Up(i) 87 | } 88 | 89 | class SECDEDCode extends Code 90 | { 91 | private val sec = new SECCode 92 | private val par = new ParityCode 93 | 94 | def width(k: Int) = sec.width(k)+1 95 | def encode(x: UInt) = par.encode(sec.encode(x)) 96 | def decode(x: UInt) = new Decoding { 97 | val secdec = sec.decode(x(x.getWidth-2,0)) 98 | val pardec = par.decode(x) 99 | 100 | def uncorrected = secdec.uncorrected 101 | def corrected = secdec.corrected 102 | def correctable = pardec.uncorrectable 103 | def uncorrectable = !pardec.uncorrectable && secdec.correctable 104 | } 105 | } 106 | 107 | object ErrGen 108 | { 109 | // generate a 1-bit error with approximate probability 2^-f 110 | def apply(width: Int, f: Int): UInt = { 111 | require(width > 0 && f >= 0 && log2Up(width) + f <= 16) 112 | UIntToOH(LFSR16()(log2Up(width)+f-1,0))(width-1,0) 113 | } 114 | def apply(x: UInt, f: Int): UInt = x ^ apply(x.getWidth, f) 115 | } 116 | 117 | class SECDEDTest extends Module 118 | { 119 | val code = new SECDEDCode 120 | val k = 4 121 | val n = code.width(k) 122 | 123 | val io = new Bundle { 124 | val original = Bits(OUTPUT, k) 125 | val encoded = Bits(OUTPUT, n) 126 | val injected = Bits(OUTPUT, n) 127 | val uncorrected = Bits(OUTPUT, k) 128 | val corrected = Bits(OUTPUT, k) 129 | val correctable = Bool(OUTPUT) 130 | val uncorrectable = Bool(OUTPUT) 131 | } 132 | 133 | val c = Counter(Bool(true), 1 << k) 134 | val numErrors = Counter(c._2, 3)._1 135 | val e = code.encode(c._1) 136 | val i = e ^ Mux(numErrors < UInt(1), UInt(0), ErrGen(n, 1)) ^ Mux(numErrors < UInt(2), UInt(0), ErrGen(n, 1)) 137 | val d = code.decode(i) 138 | 139 | io.original := c._1 140 | io.encoded := e 141 | io.injected := i 142 | io.uncorrected := d.uncorrected 143 | io.corrected := d.corrected 144 | io.correctable := d.correctable 145 | io.uncorrectable := d.uncorrectable 146 | } 147 | -------------------------------------------------------------------------------- /src/main/scala/htif.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | 5 | import Chisel._ 6 | import Chisel.ImplicitConversions._ 7 | import junctions._ 8 | import junctions.NastiConstants._ 9 | import cde.{Parameters, Field} 10 | 11 | case object HtifKey extends Field[HtifParameters] 12 | 13 | case class HtifParameters(width: Int, nCores: Int, offsetBits: Int, csrDataBits: Int, nSCR: Int = 64) 14 | 15 | trait HasHtifParameters { 16 | implicit val p: Parameters 17 | val htifExternal = p(HtifKey) 18 | val dataBits = p(TLKey(p(TLId))).dataBitsPerBeat 19 | val dataBeats = p(TLKey(p(TLId))).dataBeats 20 | val w = htifExternal.width 21 | val nSCR = htifExternal.nSCR 22 | val scrAddrBits = log2Up(nSCR) 23 | val scrDataBits = 64 24 | val scrDataBytes = scrDataBits / 8 25 | val csrDataBits = htifExternal.csrDataBits 26 | val csrDataBytes = csrDataBits / 8 27 | val offsetBits = htifExternal.offsetBits 28 | val nCores = htifExternal.nCores 29 | } 30 | 31 | abstract class HtifModule(implicit val p: Parameters) extends Module with HasHtifParameters 32 | abstract class HtifBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) 33 | with HasHtifParameters 34 | 35 | class HostIO(w: Int) extends Bundle { 36 | val clk = Bool(OUTPUT) 37 | val clk_edge = Bool(OUTPUT) 38 | val in = Decoupled(Bits(width = w)).flip 39 | val out = Decoupled(Bits(width = w)) 40 | 41 | override def cloneType = new HostIO(w).asInstanceOf[this.type] 42 | } 43 | 44 | class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { 45 | val reset = Bool(INPUT) 46 | val id = UInt(INPUT, log2Up(nCores)) 47 | val csr = new SmiIO(csrDataBits, 12).flip 48 | } 49 | 50 | class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHtifParameters { 51 | val io = new Bundle { 52 | val host = new HostIO(w) 53 | val cpu = Vec(nCores, new HtifIO).flip 54 | val mem = new ClientUncachedTileLinkIO 55 | val scr = new SmiIO(scrDataBits, scrAddrBits) 56 | } 57 | 58 | val short_request_bits = 64 59 | val long_request_bits = short_request_bits + dataBits*dataBeats 60 | require(short_request_bits % w == 0) 61 | 62 | val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits 63 | val rx_count = Reg(init=UInt(0,rx_count_w)) 64 | val rx_shifter = Reg(Bits(width = short_request_bits)) 65 | val rx_shifter_in = Cat(io.host.in.bits, rx_shifter(short_request_bits-1,w)) 66 | val next_cmd = rx_shifter_in(3,0) 67 | val cmd = Reg(Bits()) 68 | val size = Reg(Bits()) 69 | val pos = Reg(Bits()) 70 | val seqno = Reg(Bits()) 71 | val addr = Reg(Bits()) 72 | when (io.host.in.valid && io.host.in.ready) { 73 | rx_shifter := rx_shifter_in 74 | rx_count := rx_count + UInt(1) 75 | when (rx_count === UInt(short_request_bits/w-1)) { 76 | cmd := next_cmd 77 | size := rx_shifter_in(15,4) 78 | pos := rx_shifter_in(15,4+offsetBits-3) 79 | seqno := rx_shifter_in(23,16) 80 | addr := rx_shifter_in(63,24) 81 | } 82 | } 83 | 84 | val rx_word_count = (rx_count >> log2Up(short_request_bits/w)) 85 | val rx_word_done = io.host.in.valid && rx_count(log2Up(short_request_bits/w)-1,0).andR 86 | val packet_ram_depth = long_request_bits/short_request_bits-1 87 | val packet_ram = Mem(packet_ram_depth, Bits(width = short_request_bits)) 88 | when (rx_word_done && io.host.in.ready) { 89 | packet_ram(rx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1)) := rx_shifter_in 90 | } 91 | 92 | val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) 93 | 94 | val csr_addr = addr(io.cpu(0).csr.req.bits.addr.getWidth-1, 0) 95 | val csr_coreid = addr(log2Up(nCores)-1+20+1,20) 96 | val csr_wdata = packet_ram(0) 97 | 98 | val bad_mem_packet = size(offsetBits-1-3,0).orR || addr(offsetBits-1-3,0).orR 99 | val nack = Mux(cmd === cmd_readmem || cmd === cmd_writemem, bad_mem_packet, 100 | Mux(cmd === cmd_readcr || cmd === cmd_writecr, size =/= UInt(1), 101 | Bool(true))) 102 | 103 | val tx_count = Reg(init=UInt(0, rx_count_w)) 104 | val tx_subword_count = tx_count(log2Up(short_request_bits/w)-1,0) 105 | val tx_word_count = tx_count(rx_count_w-1, log2Up(short_request_bits/w)) 106 | val packet_ram_raddr = tx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1) 107 | when (io.host.out.valid && io.host.out.ready) { 108 | tx_count := tx_count + UInt(1) 109 | } 110 | 111 | val rx_done = rx_word_done && Mux(rx_word_count === UInt(0), next_cmd =/= cmd_writemem && next_cmd =/= cmd_writecr, rx_word_count === size || rx_word_count(log2Up(packet_ram_depth)-1,0) === UInt(0)) 112 | val tx_size = Mux(!nack && (cmd === cmd_readmem || cmd === cmd_readcr || cmd === cmd_writecr), size, UInt(0)) 113 | val tx_done = io.host.out.ready && tx_subword_count.andR && (tx_word_count === tx_size || tx_word_count > UInt(0) && packet_ram_raddr.andR) 114 | 115 | val state_rx :: state_csr_req :: state_csr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_tx :: Nil = Enum(UInt(), 8) 116 | val state = Reg(init=state_rx) 117 | 118 | val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || 119 | (state === state_mem_rresp && io.mem.grant.valid), dataBeats) 120 | val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) 121 | when (state === state_rx && rx_done) { 122 | state := Mux(rx_cmd === cmd_readmem, state_mem_rreq, 123 | Mux(rx_cmd === cmd_writemem, state_mem_wreq, 124 | Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_csr_req, 125 | state_tx))) 126 | } 127 | when (state === state_mem_wreq) { 128 | when (cnt_done) { state := state_mem_wresp } 129 | } 130 | when (state === state_mem_rreq) { 131 | when(io.mem.acquire.ready) { state := state_mem_rresp } 132 | } 133 | when (state === state_mem_wresp && io.mem.grant.valid) { 134 | state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) 135 | pos := pos - UInt(1) 136 | addr := addr + UInt(1 << offsetBits-3) 137 | } 138 | when (state === state_mem_rresp && cnt_done) { 139 | state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) 140 | pos := pos - UInt(1) 141 | addr := addr + UInt(1 << offsetBits-3) 142 | } 143 | when (state === state_tx && tx_done) { 144 | when (tx_word_count === tx_size) { 145 | rx_count := UInt(0) 146 | tx_count := UInt(0) 147 | } 148 | state := Mux(cmd === cmd_readmem && pos =/= UInt(0), state_mem_rreq, state_rx) 149 | } 150 | 151 | val n = dataBits/short_request_bits 152 | val mem_req_data = (0 until n).map { i => 153 | val ui = UInt(i, log2Up(n)) 154 | when (state === state_mem_rresp && io.mem.grant.valid) { 155 | packet_ram(Cat(io.mem.grant.bits.addr_beat, ui)) := 156 | io.mem.grant.bits.data((i+1)*short_request_bits-1, i*short_request_bits) 157 | } 158 | packet_ram(Cat(cnt, ui)) 159 | }.reverse.reduce(_##_) 160 | 161 | val init_addr = addr.toUInt >> (offsetBits-3) 162 | io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq 163 | io.mem.acquire.bits := Mux(cmd === cmd_writemem, 164 | PutBlock( 165 | addr_block = init_addr, 166 | addr_beat = cnt, 167 | client_xact_id = UInt(0), 168 | data = mem_req_data), 169 | GetBlock(addr_block = init_addr)) 170 | io.mem.grant.ready := Bool(true) 171 | 172 | val csrReadData = Reg(Bits(width = io.cpu(0).csr.resp.bits.getWidth)) 173 | for (i <- 0 until nCores) { 174 | val my_reset = Reg(init=Bool(true)) 175 | 176 | val cpu = io.cpu(i) 177 | val me = csr_coreid === UInt(i) 178 | cpu.csr.req.valid := state === state_csr_req && me && csr_addr =/= UInt(csr_RESET) 179 | cpu.csr.req.bits.rw := cmd === cmd_writecr 180 | cpu.csr.req.bits.addr := csr_addr 181 | cpu.csr.req.bits.data := csr_wdata 182 | cpu.reset := my_reset 183 | 184 | when (cpu.csr.req.fire()) { state := state_csr_resp } 185 | 186 | when (state === state_csr_req && me && csr_addr === UInt(csr_RESET)) { 187 | when (cmd === cmd_writecr) { 188 | my_reset := csr_wdata(0) 189 | } 190 | csrReadData := my_reset.toBits 191 | state := state_tx 192 | } 193 | 194 | cpu.csr.resp.ready := Bool(true) 195 | when (state === state_csr_resp && cpu.csr.resp.valid) { 196 | csrReadData := cpu.csr.resp.bits 197 | state := state_tx 198 | } 199 | } 200 | 201 | io.scr.req.valid := (state === state_csr_req && csr_coreid.andR) 202 | io.scr.req.bits.addr := addr(scrAddrBits - 1, 0).toUInt 203 | io.scr.req.bits.data := csr_wdata 204 | io.scr.req.bits.rw := (cmd === cmd_writecr) 205 | io.scr.resp.ready := Bool(true) 206 | 207 | when (io.scr.req.fire()) { state := state_csr_resp } 208 | when (state === state_csr_resp && io.scr.resp.valid) { 209 | csrReadData := io.scr.resp.bits 210 | state := state_tx 211 | } 212 | 213 | val tx_cmd = Mux(nack, cmd_nack, cmd_ack) 214 | val tx_cmd_ext = Cat(Bits(0, 4-tx_cmd.getWidth), tx_cmd) 215 | val tx_header = Cat(addr, seqno, tx_size, tx_cmd_ext) 216 | val tx_data = Mux(tx_word_count === UInt(0), tx_header, 217 | Mux(cmd === cmd_readcr || cmd === cmd_writecr, csrReadData, 218 | packet_ram(packet_ram_raddr))) 219 | 220 | io.host.in.ready := state === state_rx 221 | io.host.out.valid := state === state_tx 222 | io.host.out.bits := tx_data >> Cat(tx_count(log2Up(short_request_bits/w)-1,0), Bits(0, log2Up(w))) 223 | } 224 | 225 | class NastiIOHostIOConverter(htifW: Int)(implicit val p: Parameters) 226 | extends Module with HasNastiParameters { 227 | val io = new Bundle { 228 | val nasti = (new NastiIO).flip 229 | val host = new HostIO(htifW).flip 230 | val reset = Bool(OUTPUT) 231 | } 232 | 233 | def cloneType = new NastiIOHostIOConverter(htifW).asInstanceOf[this.type] 234 | 235 | val raddr = io.nasti.ar.bits.addr(6, 2) 236 | val waddr = io.nasti.aw.bits.addr(6, 2) 237 | 238 | 239 | val DCOUNT_ADDR = 0x00 240 | val RFIFO_ADDR = 0x01 241 | 242 | val WFIFO_ADDR = 0x00 243 | val RESET_ADDR = 0x1f 244 | 245 | val FIFO_DEPTH = 32 246 | 247 | val fifo_ren = Reg(init = Bool(false)) 248 | val fifo_wen = Reg(init = Bool(false)) 249 | 250 | val fifo_rd_len = Reg(UInt(width = nastiXLenBits)) 251 | val fifo_rd_id = Reg(UInt(width = nastiXIdBits)) 252 | val fifo_wr_id = Reg(UInt(width = nastiXIdBits)) 253 | val fifo_wr_ack = Reg(init = Bool(false)) 254 | 255 | val rd_count = Reg(init = Bool(false)) 256 | val wr_reset = Reg(init = Bool(false)) 257 | 258 | when (io.nasti.ar.fire()) { 259 | fifo_rd_len := io.nasti.ar.bits.len 260 | fifo_rd_id := io.nasti.ar.bits.id 261 | when (raddr === UInt(RFIFO_ADDR)) { 262 | fifo_ren := Bool(true) 263 | } .elsewhen (raddr === UInt(DCOUNT_ADDR)) { 264 | rd_count := Bool(true) 265 | } 266 | } 267 | 268 | when (io.nasti.r.fire()) { 269 | when (io.nasti.r.bits.last) { 270 | fifo_ren := Bool(false) 271 | rd_count := Bool(false) 272 | } .otherwise { fifo_rd_len := fifo_rd_len - UInt(1) } 273 | } 274 | 275 | when (io.nasti.aw.fire()) { 276 | fifo_wr_id := io.nasti.aw.bits.id 277 | when (waddr === UInt(WFIFO_ADDR)) { 278 | fifo_wen := Bool(true) 279 | } .elsewhen (waddr === UInt(RESET_ADDR)) { 280 | wr_reset := Bool(true) 281 | } 282 | } 283 | 284 | when (io.nasti.w.fire() && io.nasti.w.bits.last) { 285 | fifo_wen := Bool(false) 286 | wr_reset := Bool(false) 287 | fifo_wr_ack := Bool(true) 288 | } 289 | 290 | when (io.nasti.b.fire()) { fifo_wr_ack := Bool(false) } 291 | 292 | io.nasti.ar.ready := !fifo_ren 293 | io.nasti.aw.ready := !fifo_wen && !fifo_wr_ack 294 | io.nasti.b.valid := fifo_wr_ack 295 | io.nasti.b.bits := NastiWriteResponseChannel(id = fifo_wr_id) 296 | 297 | io.reset := io.nasti.w.valid && wr_reset 298 | 299 | val hn_fifo = Module(new MultiWidthFifo(htifW, nastiXDataBits, FIFO_DEPTH)) 300 | hn_fifo.io.in <> io.host.out 301 | hn_fifo.io.out.ready := fifo_ren && io.nasti.r.ready 302 | io.nasti.r.valid := (fifo_ren && hn_fifo.io.out.valid) || rd_count 303 | io.nasti.r.bits := NastiReadDataChannel( 304 | id = fifo_rd_id, 305 | data = Mux(fifo_ren, hn_fifo.io.out.bits, hn_fifo.io.count), 306 | last = (fifo_rd_len === UInt(0))) 307 | 308 | val nh_fifo = Module(new MultiWidthFifo(nastiXDataBits, htifW, FIFO_DEPTH)) 309 | io.host.in <> nh_fifo.io.out 310 | nh_fifo.io.in.valid := fifo_wen && io.nasti.w.valid 311 | nh_fifo.io.in.bits := io.nasti.w.bits.data 312 | io.nasti.w.ready := (fifo_wen && nh_fifo.io.in.ready) || wr_reset 313 | 314 | assert(!io.nasti.w.valid || io.nasti.w.bits.strb.andR, 315 | "Nasti to HostIO converter cannot take partial writes") 316 | assert(!io.nasti.ar.valid || 317 | io.nasti.ar.bits.len === UInt(0) || 318 | io.nasti.ar.bits.burst === BURST_FIXED, 319 | "Nasti to HostIO converter can only take fixed bursts") 320 | assert(!io.nasti.aw.valid || 321 | io.nasti.aw.bits.len === UInt(0) || 322 | io.nasti.aw.bits.burst === BURST_FIXED, 323 | "Nasti to HostIO converter can only take fixed bursts") 324 | } 325 | -------------------------------------------------------------------------------- /src/main/scala/interconnect.scala: -------------------------------------------------------------------------------- 1 | package uncore 2 | 3 | import Chisel._ 4 | import junctions._ 5 | import scala.collection.mutable.ArraySeq 6 | import cde.{Parameters, Field} 7 | 8 | 9 | /** PortedTileLinkNetworks combine a TileLink protocol with a particular physical 10 | * network implementation. 11 | * 12 | * Specifically, they provide mappings between ClientTileLinkIO/ 13 | * ManagerTileLinkIO channels and LogicalNetwork ports (i.e. generic 14 | * TileLinkIO with networking headers). Channels coming into the network have 15 | * appropriate networking headers appended and outgoing channels have their 16 | * headers stripped. 17 | * 18 | * @constructor base class constructor for Ported TileLink NoC 19 | * @param addrToManagerId a mapping from a physical address to the network 20 | * id of a coherence manager 21 | * @param sharerToClientId a mapping from the id of a particular coherent 22 | * client (as determined by e.g. the directory) and the network id 23 | * of that client 24 | * @param clientDepths the depths of the queue that should be used to buffer 25 | * each channel on the client side of the network 26 | * @param managerDepths the depths of the queue that should be used to buffer 27 | * each channel on the manager side of the network 28 | */ 29 | abstract class PortedTileLinkNetwork( 30 | addrToManagerId: UInt => UInt, 31 | sharerToClientId: UInt => UInt, 32 | clientDepths: TileLinkDepths, 33 | managerDepths: TileLinkDepths) 34 | (implicit p: Parameters) extends TLModule()(p) { 35 | val nClients = tlNClients 36 | val nManagers = tlNManagers 37 | val io = new Bundle { 38 | val clients_cached = Vec(tlNCachingClients, new ClientTileLinkIO).flip 39 | val clients_uncached = Vec(tlNCachelessClients, new ClientUncachedTileLinkIO).flip 40 | val managers = Vec(nManagers, new ManagerTileLinkIO).flip 41 | } 42 | 43 | val clients = (io.clients_cached ++ io.clients_uncached).zipWithIndex.map { 44 | case (io, idx) => { 45 | val qs = Module(new TileLinkEnqueuer(clientDepths)) 46 | io match { 47 | case c: ClientTileLinkIO => { 48 | val port = Module(new ClientTileLinkNetworkPort(idx, addrToManagerId)) 49 | port.io.client <> c 50 | qs.io.client <> port.io.network 51 | qs.io.manager 52 | } 53 | case u: ClientUncachedTileLinkIO => { 54 | val port = Module(new ClientUncachedTileLinkNetworkPort(idx, addrToManagerId)) 55 | port.io.client <> u 56 | qs.io.client <> port.io.network 57 | qs.io.manager 58 | } 59 | } 60 | } 61 | } 62 | 63 | val managers = io.managers.zipWithIndex.map { 64 | case (m, i) => { 65 | val port = Module(new ManagerTileLinkNetworkPort(i, sharerToClientId)) 66 | val qs = Module(new TileLinkEnqueuer(managerDepths)) 67 | port.io.manager <> m 68 | port.io.network <> qs.io.manager 69 | qs.io.client 70 | } 71 | } 72 | } 73 | 74 | /** A simple arbiter for each channel that also deals with header-based routing. 75 | * Assumes a single manager agent. */ 76 | class PortedTileLinkArbiter( 77 | sharerToClientId: UInt => UInt = (u: UInt) => u, 78 | clientDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0), 79 | managerDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0)) 80 | (implicit p: Parameters) 81 | extends PortedTileLinkNetwork(u => UInt(0), sharerToClientId, clientDepths, managerDepths)(p) 82 | with TileLinkArbiterLike 83 | with PassesId { 84 | val arbN = nClients 85 | require(nManagers == 1) 86 | if(arbN > 1) { 87 | hookupClientSource(clients.map(_.acquire), managers.head.acquire) 88 | hookupClientSource(clients.map(_.release), managers.head.release) 89 | hookupFinish(clients.map(_.finish), managers.head.finish) 90 | hookupManagerSourceWithHeader(clients.map(_.probe), managers.head.probe) 91 | hookupManagerSourceWithHeader(clients.map(_.grant), managers.head.grant) 92 | } else { 93 | managers.head <> clients.head 94 | } 95 | } 96 | 97 | /** Provides a separate physical crossbar for each channel. Assumes multiple manager 98 | * agents. Managers are assigned to higher physical network port ids than 99 | * clients, and translations between logical network id and physical crossbar 100 | * port id are done automatically. 101 | */ 102 | class PortedTileLinkCrossbar( 103 | addrToManagerId: UInt => UInt = u => UInt(0), 104 | sharerToClientId: UInt => UInt = u => u, 105 | clientDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0), 106 | managerDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0)) 107 | (implicit p: Parameters) 108 | extends PortedTileLinkNetwork(addrToManagerId, sharerToClientId, clientDepths, managerDepths)(p) { 109 | val n = p(LNEndpoints) 110 | val phyHdrWidth = log2Up(n) 111 | val count = tlDataBeats 112 | // Actually instantiate the particular networks required for TileLink 113 | val acqNet = Module(new BasicCrossbar(n, new Acquire, count, Some((a: PhysicalNetworkIO[Acquire]) => a.payload.hasMultibeatData()))) 114 | val relNet = Module(new BasicCrossbar(n, new Release, count, Some((r: PhysicalNetworkIO[Release]) => r.payload.hasMultibeatData()))) 115 | val prbNet = Module(new BasicCrossbar(n, new Probe)) 116 | val gntNet = Module(new BasicCrossbar(n, new Grant, count, Some((g: PhysicalNetworkIO[Grant]) => g.payload.hasMultibeatData()))) 117 | val ackNet = Module(new BasicCrossbar(n, new Finish)) 118 | 119 | // Aliases for the various network IO bundle types 120 | type PNIO[T <: Data] = DecoupledIO[PhysicalNetworkIO[T]] 121 | type LNIO[T <: Data] = DecoupledIO[LogicalNetworkIO[T]] 122 | type FromCrossbar[T <: Data] = PNIO[T] => LNIO[T] 123 | type ToCrossbar[T <: Data] = LNIO[T] => PNIO[T] 124 | 125 | // Shims for converting between logical network IOs and physical network IOs 126 | def crossbarToManagerShim[T <: Data](in: PNIO[T]): LNIO[T] = { 127 | val out = DefaultFromPhysicalShim(in) 128 | out.bits.header.src := in.bits.header.src - UInt(nManagers) 129 | out 130 | } 131 | def crossbarToClientShim[T <: Data](in: PNIO[T]): LNIO[T] = { 132 | val out = DefaultFromPhysicalShim(in) 133 | out.bits.header.dst := in.bits.header.dst - UInt(nManagers) 134 | out 135 | } 136 | def managerToCrossbarShim[T <: Data](in: LNIO[T]): PNIO[T] = { 137 | val out = DefaultToPhysicalShim(n, in) 138 | out.bits.header.dst := in.bits.header.dst + UInt(nManagers, phyHdrWidth) 139 | out 140 | } 141 | def clientToCrossbarShim[T <: Data](in: LNIO[T]): PNIO[T] = { 142 | val out = DefaultToPhysicalShim(n, in) 143 | out.bits.header.src := in.bits.header.src + UInt(nManagers, phyHdrWidth) 144 | out 145 | } 146 | 147 | // Make an individual connection between virtual and physical ports using 148 | // a particular shim. Also pin the unused Decoupled control signal low. 149 | def doDecoupledInputHookup[T <: Data](phys_in: PNIO[T], phys_out: PNIO[T], log_io: LNIO[T], shim: ToCrossbar[T]) = { 150 | val s = shim(log_io) 151 | phys_in.valid := s.valid 152 | phys_in.bits := s.bits 153 | s.ready := phys_in.ready 154 | phys_out.ready := Bool(false) 155 | } 156 | 157 | def doDecoupledOutputHookup[T <: Data](phys_in: PNIO[T], phys_out: PNIO[T], log_io: LNIO[T], shim: FromCrossbar[T]) = { 158 | val s = shim(phys_out) 159 | log_io.valid := s.valid 160 | log_io.bits := s.bits 161 | s.ready := log_io.ready 162 | phys_in.valid := Bool(false) 163 | } 164 | 165 | //Hookup all instances of a particular subbundle of TileLink 166 | def doDecoupledHookups[T <: Data](physIO: BasicCrossbarIO[T], getLogIO: TileLinkIO => LNIO[T]) = { 167 | physIO.in.head.bits.payload match { 168 | case c: ClientToManagerChannel => { 169 | managers.zipWithIndex.map { case (i, id) => 170 | doDecoupledOutputHookup(physIO.in(id), physIO.out(id), getLogIO(i), crossbarToManagerShim[T]) 171 | } 172 | clients.zipWithIndex.map { case (i, id) => 173 | doDecoupledInputHookup(physIO.in(id+nManagers), physIO.out(id+nManagers), getLogIO(i), clientToCrossbarShim[T]) 174 | } 175 | } 176 | case m: ManagerToClientChannel => { 177 | managers.zipWithIndex.map { case (i, id) => 178 | doDecoupledInputHookup(physIO.in(id), physIO.out(id), getLogIO(i), managerToCrossbarShim[T]) 179 | } 180 | clients.zipWithIndex.map { case (i, id) => 181 | doDecoupledOutputHookup(physIO.in(id+nManagers), physIO.out(id+nManagers), getLogIO(i), crossbarToClientShim[T]) 182 | } 183 | } 184 | } 185 | } 186 | 187 | doDecoupledHookups(acqNet.io, (tl: TileLinkIO) => tl.acquire) 188 | doDecoupledHookups(relNet.io, (tl: TileLinkIO) => tl.release) 189 | doDecoupledHookups(prbNet.io, (tl: TileLinkIO) => tl.probe) 190 | doDecoupledHookups(gntNet.io, (tl: TileLinkIO) => tl.grant) 191 | doDecoupledHookups(ackNet.io, (tl: TileLinkIO) => tl.finish) 192 | } 193 | 194 | class ClientUncachedTileLinkIORouter( 195 | nOuter: Int, routeSel: UInt => UInt)(implicit p: Parameters) 196 | extends TLModule { 197 | 198 | val io = new Bundle { 199 | val in = (new ClientUncachedTileLinkIO).flip 200 | val out = Vec(nOuter, new ClientUncachedTileLinkIO) 201 | } 202 | 203 | val acq_route = routeSel(io.in.acquire.bits.full_addr()) 204 | 205 | io.in.acquire.ready := Bool(false) 206 | 207 | io.out.zipWithIndex.foreach { case (out, i) => 208 | out.acquire.valid := io.in.acquire.valid && acq_route(i) 209 | out.acquire.bits := io.in.acquire.bits 210 | when (acq_route(i)) { io.in.acquire.ready := out.acquire.ready } 211 | } 212 | 213 | val gnt_arb = Module(new LockingRRArbiter( 214 | new Grant, nOuter, tlDataBeats, Some((gnt: Grant) => gnt.hasMultibeatData()))) 215 | gnt_arb.io.in <> io.out.map(_.grant) 216 | io.in.grant <> gnt_arb.io.out 217 | 218 | assert(!io.in.acquire.valid || acq_route.orR, "No valid route") 219 | } 220 | 221 | class TileLinkInterconnectIO(val nInner: Int, val nOuter: Int) 222 | (implicit p: Parameters) extends Bundle { 223 | val in = Vec(nInner, new ClientUncachedTileLinkIO).flip 224 | val out = Vec(nOuter, new ClientUncachedTileLinkIO) 225 | } 226 | 227 | class ClientUncachedTileLinkIOCrossbar( 228 | nInner: Int, nOuter: Int, routeSel: UInt => UInt) 229 | (implicit p: Parameters) extends TLModule { 230 | 231 | val io = new TileLinkInterconnectIO(nInner, nOuter) 232 | 233 | if (nInner == 1) { 234 | val router = Module(new ClientUncachedTileLinkIORouter(nOuter, routeSel)) 235 | router.io.in <> io.in.head 236 | io.out <> router.io.out 237 | } else { 238 | val routers = List.fill(nInner) { 239 | Module(new ClientUncachedTileLinkIORouter(nOuter, routeSel)) } 240 | val arbiters = List.fill(nOuter) { 241 | Module(new ClientUncachedTileLinkIOArbiter(nInner)) } 242 | 243 | for (i <- 0 until nInner) { 244 | routers(i).io.in <> io.in(i) 245 | } 246 | 247 | for (i <- 0 until nOuter) { 248 | arbiters(i).io.in <> routers.map(r => r.io.out(i)) 249 | io.out(i) <> arbiters(i).io.out 250 | } 251 | } 252 | } 253 | 254 | abstract class TileLinkInterconnect(implicit p: Parameters) extends TLModule()(p) { 255 | val nInner: Int 256 | val nOuter: Int 257 | 258 | lazy val io = new TileLinkInterconnectIO(nInner, nOuter) 259 | } 260 | 261 | class TileLinkRecursiveInterconnect( 262 | val nInner: Int, addrmap: AddrMap, base: BigInt) 263 | (implicit p: Parameters) extends TileLinkInterconnect()(p) { 264 | val levelSize = addrmap.size 265 | val nOuter = addrmap.countSlaves 266 | 267 | val addrHashMap = new AddrHashMap(addrmap, base) 268 | val routeSel = (addr: UInt) => { 269 | Cat(addrmap.map { case entry => 270 | val hashEntry = addrHashMap(entry.name) 271 | addr >= UInt(hashEntry.start) && addr < UInt(hashEntry.start + hashEntry.region.size) 272 | }.reverse) 273 | } 274 | 275 | val xbar = Module(new ClientUncachedTileLinkIOCrossbar(nInner, levelSize, routeSel)) 276 | xbar.io.in <> io.in 277 | 278 | io.out <> addrmap.zip(xbar.io.out).flatMap { 279 | case (entry, xbarOut) => { 280 | entry.region match { 281 | case _: MemSize => 282 | Some(xbarOut) 283 | case MemSubmap(_, submap, _) if submap.isEmpty => 284 | xbarOut.acquire.ready := Bool(false) 285 | xbarOut.grant.valid := Bool(false) 286 | None 287 | case MemSubmap(_, submap, sharePort) if sharePort => 288 | Some(xbarOut) 289 | case MemSubmap(_, submap, _) => 290 | val ic = Module(new TileLinkRecursiveInterconnect(1, submap, addrHashMap(entry.name).start)) 291 | ic.io.in.head <> xbarOut 292 | ic.io.out 293 | } 294 | } 295 | } 296 | } 297 | 298 | class TileLinkMemoryInterconnect( 299 | nBanksPerChannel: Int, nChannels: Int) 300 | (implicit p: Parameters) extends TileLinkInterconnect()(p) { 301 | 302 | val nBanks = nBanksPerChannel * nChannels 303 | val nInner = nBanks 304 | val nOuter = nChannels 305 | 306 | def connectChannel(outer: ClientUncachedTileLinkIO, inner: ClientUncachedTileLinkIO) { 307 | outer <> inner 308 | outer.acquire.bits.addr_block := inner.acquire.bits.addr_block >> UInt(log2Ceil(nChannels)) 309 | } 310 | 311 | for (i <- 0 until nChannels) { 312 | /* Bank assignments to channels are strided so that consecutive banks 313 | * map to different channels. That way, consecutive cache lines also 314 | * map to different channels */ 315 | val banks = (i until nBanks by nChannels).map(j => io.in(j)) 316 | 317 | val channelArb = Module(new ClientUncachedTileLinkIOArbiter(nBanksPerChannel)) 318 | channelArb.io.in <> banks 319 | connectChannel(io.out(i), channelArb.io.out) 320 | } 321 | } 322 | 323 | /** Allows users to switch between various memory configurations. Note that 324 | * this is a dangerous operation: not only does switching the select input to 325 | * this module violate TileLink, it also causes the memory of the machine to 326 | * become garbled. It's expected that select only changes at boot time, as 327 | * part of the memory controller configuration. */ 328 | class TileLinkMemorySelectorIO(val nBanks: Int, val maxMemChannels: Int, nConfigs: Int) 329 | (implicit p: Parameters) 330 | extends TileLinkInterconnectIO(nBanks, maxMemChannels) { 331 | val select = UInt(INPUT, width = log2Up(nConfigs)) 332 | override def cloneType = 333 | new TileLinkMemorySelectorIO(nBanks, maxMemChannels, nConfigs).asInstanceOf[this.type] 334 | } 335 | 336 | class TileLinkMemorySelector(nBanks: Int, maxMemChannels: Int, configs: Seq[Int]) 337 | (implicit p: Parameters) 338 | extends TileLinkInterconnect()(p) { 339 | val nInner = nBanks 340 | val nOuter = maxMemChannels 341 | val nConfigs = configs.size 342 | 343 | override lazy val io = new TileLinkMemorySelectorIO(nBanks, maxMemChannels, nConfigs) 344 | 345 | def muxOnSelect[T <: Data](up: DecoupledIO[T], dn: DecoupledIO[T], active: Bool): Unit = { 346 | when (active) { dn.bits := up.bits } 347 | when (active) { up.ready := dn.ready } 348 | when (active) { dn.valid := up.valid } 349 | } 350 | 351 | def muxOnSelect(up: ClientUncachedTileLinkIO, dn: ClientUncachedTileLinkIO, active: Bool): Unit = { 352 | muxOnSelect(up.acquire, dn.acquire, active) 353 | muxOnSelect(dn.grant, up.grant, active) 354 | } 355 | 356 | def muxOnSelect(up: Vec[ClientUncachedTileLinkIO], dn: Vec[ClientUncachedTileLinkIO], active: Bool) : Unit = { 357 | for (i <- 0 until up.size) 358 | muxOnSelect(up(i), dn(i), active) 359 | } 360 | 361 | /* Disconnects a vector of TileLink ports, which involves setting them to 362 | * invalid. Due to Chisel reasons, we need to also set the bits to 0 (since 363 | * there can't be any unconnected inputs). */ 364 | def disconnectOuter(outer: Vec[ClientUncachedTileLinkIO]) = { 365 | outer.foreach{ m => 366 | m.acquire.valid := Bool(false) 367 | m.acquire.bits := m.acquire.bits.fromBits(UInt(0)) 368 | m.grant.ready := Bool(false) 369 | } 370 | } 371 | 372 | def disconnectInner(inner: Vec[ClientUncachedTileLinkIO]) = { 373 | inner.foreach { m => 374 | m.grant.valid := Bool(false) 375 | m.grant.bits := m.grant.bits.fromBits(UInt(0)) 376 | m.acquire.ready := Bool(false) 377 | } 378 | } 379 | 380 | /* Provides default wires on all our outputs. */ 381 | disconnectOuter(io.out) 382 | disconnectInner(io.in) 383 | 384 | /* Constructs interconnects for each of the layouts suggested by the 385 | * configuration and switches between them based on the select input. */ 386 | configs.zipWithIndex.foreach{ case (nChannels, select) => 387 | val nBanksPerChannel = nBanks / nChannels 388 | val ic = Module(new TileLinkMemoryInterconnect(nBanksPerChannel, nChannels)) 389 | disconnectInner(ic.io.out) 390 | disconnectOuter(ic.io.in) 391 | muxOnSelect(io.in, ic.io.in, io.select === UInt(select)) 392 | muxOnSelect(ic.io.out, io.out, io.select === UInt(select)) 393 | } 394 | } 395 | -------------------------------------------------------------------------------- /src/main/scala/metadata.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | import Chisel._ 5 | import cde.{Parameters, Field} 6 | 7 | /** Base class to represent coherence information in clients and managers */ 8 | abstract class CoherenceMetadata(implicit p: Parameters) extends TLBundle()(p) { 9 | val co = tlCoh 10 | } 11 | 12 | /** Stores the client-side coherence information, 13 | * such as permissions on the data and whether the data is dirty. 14 | * Its API can be used to make TileLink messages in response to 15 | * memory operations or [[uncore.Probe]] messages. 16 | */ 17 | class ClientMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { 18 | /** Actual state information stored in this bundle */ 19 | val state = UInt(width = co.clientStateWidth) 20 | 21 | /** Metadata equality */ 22 | def ===(rhs: ClientMetadata): Bool = this.state === rhs.state 23 | def =/=(rhs: ClientMetadata): Bool = !this.===(rhs) 24 | 25 | /** Is the block's data present in this cache */ 26 | def isValid(dummy: Int = 0): Bool = co.isValid(this) 27 | /** Does this cache have permissions on this block sufficient to perform op */ 28 | def isHit(op_code: UInt): Bool = co.isHit(op_code, this) 29 | /** Does this cache lack permissions on this block sufficient to perform op */ 30 | def isMiss(op_code: UInt): Bool = !co.isHit(op_code, this) 31 | /** Does a secondary miss on the block require another Acquire message */ 32 | def requiresAcquireOnSecondaryMiss(first_op: UInt, second_op: UInt): Bool = 33 | co.requiresAcquireOnSecondaryMiss(first_op, second_op, this) 34 | /** Does op require a Release to be made to outer memory */ 35 | def requiresReleaseOnCacheControl(op_code: UInt): Bool = 36 | co.requiresReleaseOnCacheControl(op_code: UInt, this) 37 | /** Does an eviction require a Release to be made to outer memory */ 38 | def requiresVoluntaryWriteback(dummy: Int = 0): Bool = 39 | co.requiresReleaseOnCacheControl(M_FLUSH, this) 40 | 41 | /** Constructs an Acquire message based on this metdata and a memory operation 42 | * 43 | * @param client_xact_id client's transaction id 44 | * @param addr_block address of the cache block 45 | * @param op_code a memory operation from [[uncore.constants.MemoryOpConstants]] 46 | */ 47 | def makeAcquire( 48 | op_code: UInt, 49 | client_xact_id: UInt, 50 | addr_block: UInt): Acquire = 51 | Acquire( 52 | is_builtin_type = Bool(false), 53 | a_type = co.getAcquireType(op_code, this), 54 | client_xact_id = client_xact_id, 55 | addr_block = addr_block, 56 | union = Cat(op_code, Bool(true)))(p) 57 | 58 | /** Constructs a Release message based on this metadata on cache control op 59 | * 60 | * @param client_xact_id client's transaction id 61 | * @param addr_block address of the cache block 62 | * @param addr_beat sub-block address (which beat) 63 | * @param data data being written back 64 | */ 65 | def makeVoluntaryRelease( 66 | op_code: UInt, 67 | client_xact_id: UInt, 68 | addr_block: UInt, 69 | addr_beat: UInt = UInt(0), 70 | data: UInt = UInt(0), 71 | tag: UInt = UInt(0)): Release = 72 | Release( 73 | voluntary = Bool(true), 74 | r_type = co.getReleaseType(op_code, this), 75 | client_xact_id = client_xact_id, 76 | addr_block = addr_block, 77 | addr_beat = addr_beat, 78 | data = data, 79 | tag = tag)(p) 80 | 81 | /** Constructs a Release message based on this metadata on an eviction 82 | * 83 | * @param client_xact_id client's transaction id 84 | * @param addr_block address of the cache block 85 | * @param addr_beat sub-block address (which beat) 86 | * @param data data being written back 87 | */ 88 | def makeVoluntaryWriteback( 89 | client_xact_id: UInt, 90 | addr_block: UInt, 91 | addr_beat: UInt = UInt(0), 92 | data: UInt = UInt(0), 93 | tag: UInt = UInt(0)): Release = 94 | makeVoluntaryRelease( 95 | op_code = M_FLUSH, 96 | client_xact_id = client_xact_id, 97 | addr_block = addr_block, 98 | addr_beat = addr_beat, 99 | data = data, 100 | tag = tag) 101 | 102 | /** Constructs a Release message based on this metadata and a [[uncore.Probe]] 103 | * 104 | * @param the incoming [[uncore.Probe]] 105 | * @param addr_beat sub-block address (which beat) 106 | * @param data data being released 107 | */ 108 | def makeRelease( 109 | prb: Probe, 110 | addr_beat: UInt = UInt(0), 111 | data: UInt = UInt(0), 112 | tag: UInt = UInt(0)): Release = 113 | Release( 114 | voluntary = Bool(false), 115 | r_type = co.getReleaseType(prb, this), 116 | client_xact_id = UInt(0), 117 | addr_block = prb.addr_block, 118 | addr_beat = addr_beat, 119 | data = data, 120 | tag = tag)(p) 121 | 122 | /** New metadata after receiving a [[uncore.Grant]] 123 | * 124 | * @param incoming the incoming [[uncore.Grant]] 125 | * @param pending the mem op that triggered this transaction 126 | */ 127 | def onGrant(incoming: Grant, pending: UInt): ClientMetadata = 128 | co.clientMetadataOnGrant(incoming, pending, this) 129 | 130 | /** New metadata after receiving a [[uncore.Probe]] 131 | * 132 | * @param incoming the incoming [[uncore.Probe]] 133 | */ 134 | def onProbe(incoming: Probe): ClientMetadata = 135 | co.clientMetadataOnProbe(incoming, this) 136 | 137 | /** New metadata after a op_code hits this block 138 | * 139 | * @param op_code a memory operation from [[uncore.constants.MemoryOpConstants]] 140 | */ 141 | def onHit(op_code: UInt): ClientMetadata = 142 | co.clientMetadataOnHit(op_code, this) 143 | 144 | /** New metadata after op_code releases permissions on this block 145 | * 146 | * @param op_code a memory operation from [[uncore.constants.MemoryOpConstants]] 147 | */ 148 | def onCacheControl(op_code: UInt): ClientMetadata = 149 | co.clientMetadataOnCacheControl(op_code, this) 150 | } 151 | 152 | /** Factories for ClientMetadata, including on reset */ 153 | object ClientMetadata { 154 | def apply(state: UInt)(implicit p: Parameters) = { 155 | val meta = Wire(new ClientMetadata) 156 | meta.state := state 157 | meta 158 | } 159 | def onReset(implicit p: Parameters) = ClientMetadata(UInt(0))(p) // TODO: assumes clientInvalid === 0 160 | } 161 | 162 | /** Stores manager-side information about the status 163 | * of a cache block, including whether it has any known sharers. 164 | * 165 | * Its API can be used to create [[uncore.Probe]] and [[uncore.Grant]] messages. 166 | */ 167 | class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { 168 | // Currently no coherence policies assume manager-side state information 169 | // val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel 170 | 171 | /** The directory information for this block */ 172 | val sharers = UInt(width = co.dir.width) 173 | 174 | /** Metadata equality */ 175 | def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel 176 | this.sharers === rhs.sharers 177 | def =/=(rhs: ManagerMetadata): Bool = !this.===(rhs) 178 | 179 | /** Converts the directory info into an N-hot sharer bitvector (i.e. full representation) */ 180 | def full(dummy: Int = 0): UInt = co.dir.full(this.sharers) 181 | 182 | /** Does this [[uncore.Acquire]] require [[uncore.Probe Probes]] to be sent */ 183 | def requiresProbes(acq: HasAcquireType): Bool = co.requiresProbes(acq, this) 184 | /** Does this memory op require [[uncore.Probe Probes]] to be sent */ 185 | def requiresProbes(op_code: UInt): Bool = co.requiresProbes(op_code, this) 186 | /** Does an eviction require [[uncore.Probe Probes]] to be sent */ 187 | def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = 188 | co.requiresProbes(M_FLUSH, this) 189 | 190 | /** Construct an appropriate [[uncore.ProbeToDst]] for a given [[uncore.Acquire]] 191 | * 192 | * @param dst Destination client id for this Probe 193 | * @param acq Acquire message triggering this Probe 194 | * @param addr_block address of the cache block being probed 195 | */ 196 | def makeProbe(dst: UInt, acq: HasAcquireType, addr_block: UInt): ProbeToDst = 197 | Probe(dst, co.getProbeType(acq, this), addr_block)(p) 198 | 199 | /** Construct an appropriate [[uncore.ProbeToDst]] for a given [[uncore.Acquire]] 200 | * 201 | * @param dst Destination client id for this Probe 202 | * @param acq Acquire message triggering this Probe 203 | */ 204 | def makeProbe(dst: UInt, acq: AcquireMetadata): ProbeToDst = 205 | Probe(dst, co.getProbeType(acq, this), acq.addr_block)(p) 206 | 207 | /** Construct an appropriate [[uncore.ProbeToDst]] for a given mem op 208 | * 209 | * @param dst Destination client id for this Probe 210 | * @param op_code memory operation triggering this Probe 211 | * @param addr_block address of the cache block being probed 212 | */ 213 | def makeProbe(dst: UInt, op_code: UInt, addr_block: UInt): ProbeToDst = 214 | Probe(dst, co.getProbeType(op_code, this), addr_block)(p) 215 | 216 | /** Construct an appropriate [[uncore.ProbeToDst]] for an eviction 217 | * 218 | * @param dst Destination client id for this Probe 219 | * @param addr_block address of the cache block being probed prior to eviction 220 | */ 221 | def makeProbeForVoluntaryWriteback(dst: UInt, addr_block: UInt): ProbeToDst = 222 | makeProbe(dst, M_FLUSH, addr_block) 223 | 224 | /** Construct an appropriate [[uncore.GrantToDst]] to acknowledge an [[uncore.Release]] 225 | * 226 | * @param rel Release message being acknowledged by this Grant 227 | */ 228 | def makeGrant(rel: ReleaseMetadata with HasClientId): GrantToDst = 229 | GrantToDst( 230 | dst = rel.client_id, 231 | is_builtin_type = Bool(true), 232 | g_type = Grant.voluntaryAckType, 233 | client_xact_id = rel.client_xact_id, 234 | manager_xact_id = UInt(0))(p) 235 | 236 | /** Construct an appropriate [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] 237 | * 238 | * May contain single or multiple beats of data, or just be a permissions upgrade. 239 | * 240 | * @param acq Acquire message being responded to by this Grant 241 | * @param manager_xact_id manager's transaction id 242 | * @param addr_beat beat id of the data 243 | * @param data data being refilled to the original requestor 244 | */ 245 | def makeGrant( 246 | acq: AcquireMetadata with HasClientId, 247 | manager_xact_id: UInt, 248 | addr_beat: UInt = UInt(0), 249 | data: UInt = UInt(0), 250 | tag: UInt = UInt(0)): GrantToDst = 251 | GrantToDst( 252 | dst = acq.client_id, 253 | is_builtin_type = acq.isBuiltInType(), 254 | g_type = co.getGrantType(acq, this), 255 | client_xact_id = acq.client_xact_id, 256 | manager_xact_id = manager_xact_id, 257 | addr_beat = addr_beat, 258 | data = data, 259 | tag = tag)(p) 260 | 261 | /** Construct an [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] with some overrides 262 | * 263 | * Used to respond to secondary misses merged into this transaction. 264 | * May contain single or multiple beats of data. 265 | * 266 | * @param sec Secondary miss info 267 | * @param manager_xact_id manager's transaction id 268 | * @param data data being refilled to the original requestor 269 | */ 270 | def makeGrant( 271 | sec: SecondaryMissInfo, 272 | manager_xact_id: UInt, 273 | data: UInt, 274 | tag: UInt): GrantToDst = { 275 | GrantToDst( 276 | dst = sec.client_id, 277 | is_builtin_type = sec.isBuiltInType(), 278 | g_type = co.getGrantType(sec, this), 279 | client_xact_id = sec.client_xact_id, 280 | manager_xact_id = manager_xact_id, 281 | addr_beat = sec.addr_beat, 282 | data = data, 283 | tag = tag)(p) 284 | } 285 | 286 | /** New metadata after receiving a [[uncore.ReleaseFromSrc]] 287 | * 288 | * @param incoming the incoming [[uncore.ReleaseFromSrc]] 289 | */ 290 | def onRelease(incoming: ReleaseMetadata with HasClientId): ManagerMetadata = 291 | co.managerMetadataOnRelease(incoming, incoming.client_id, this) 292 | 293 | /** New metadata after sending a [[uncore.GrantToDst]] 294 | * 295 | * @param outgoing the outgoing [[uncore.GrantToDst]] 296 | */ 297 | def onGrant(outgoing: GrantMetadata with HasClientId): ManagerMetadata = 298 | co.managerMetadataOnGrant(outgoing, outgoing.client_id, this) 299 | } 300 | 301 | /** Factories for ManagerMetadata, including on reset */ 302 | object ManagerMetadata { 303 | def apply(sharers: UInt, state: UInt = UInt(width = 0))(implicit p: Parameters) = { 304 | val meta = Wire(new ManagerMetadata) 305 | //meta.state := state TODO: Fix 0-width wires in Chisel 306 | meta.sharers := sharers 307 | meta 308 | } 309 | def apply(implicit p: Parameters) = { 310 | val meta = Wire(new ManagerMetadata) 311 | //meta.state := UInt(width = 0) TODO: Fix 0-width wires in Chisel 312 | meta.sharers := meta.co.dir.flush 313 | meta 314 | } 315 | def onReset(implicit p: Parameters) = ManagerMetadata(p) 316 | } 317 | 318 | /** HierarchicalMetadata is used in a cache in a multi-level memory hierarchy 319 | * that is a manager with respect to some inner caches and a client with 320 | * respect to some outer cache. 321 | * 322 | * This class makes use of two different sets of TileLink parameters, which are 323 | * applied by contextually mapping [[uncore.TLId]] to one of 324 | * [[uncore.InnerTLId]] or [[uncore.OuterTLId]]. 325 | */ 326 | class HierarchicalMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { 327 | val inner: ManagerMetadata = new ManagerMetadata()(p.alterPartial({case TLId => p(InnerTLId)})) 328 | val outer: ClientMetadata = new ClientMetadata()(p.alterPartial({case TLId => p(OuterTLId)})) 329 | def ===(rhs: HierarchicalMetadata): Bool = 330 | this.inner === rhs.inner && this.outer === rhs.outer 331 | def =/=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) 332 | } 333 | 334 | /** Factories for HierarchicalMetadata, including on reset */ 335 | object HierarchicalMetadata { 336 | def apply(inner: ManagerMetadata, outer: ClientMetadata) 337 | (implicit p: Parameters): HierarchicalMetadata = { 338 | val m = Wire(new HierarchicalMetadata) 339 | m.inner := inner 340 | m.outer := outer 341 | m 342 | } 343 | def onReset(implicit p: Parameters): HierarchicalMetadata = 344 | apply(ManagerMetadata.onReset, ClientMetadata.onReset) 345 | } 346 | -------------------------------------------------------------------------------- /src/main/scala/network.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | import Chisel._ 5 | import cde.{Parameters, Field} 6 | 7 | case object LNEndpoints extends Field[Int] 8 | case object LNHeaderBits extends Field[Int] 9 | 10 | class PhysicalHeader(n: Int) extends Bundle { 11 | val src = UInt(width = log2Up(n)) 12 | val dst = UInt(width = log2Up(n)) 13 | } 14 | 15 | class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { 16 | val header = new PhysicalHeader(n) 17 | val payload = dType.cloneType 18 | override def cloneType = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] 19 | } 20 | 21 | class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { 22 | val in = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))).flip 23 | val out = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))) 24 | } 25 | 26 | abstract class PhysicalNetwork extends Module 27 | 28 | class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) extends PhysicalNetwork { 29 | val io = new BasicCrossbarIO(n, dType) 30 | 31 | io.in.foreach { _.ready := Bool(false) } 32 | 33 | io.out.zipWithIndex.map{ case (out, i) => { 34 | val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count, needsLock)) 35 | (rrarb.io.in, io.in).zipped.map{ case (arb, in) => { 36 | val destined = in.bits.header.dst === UInt(i) 37 | arb.valid := in.valid && destined 38 | arb.bits := in.bits 39 | when (arb.ready && destined) { in.ready := Bool(true) } 40 | }} 41 | out <> rrarb.io.out 42 | }} 43 | } 44 | 45 | abstract class LogicalNetwork extends Module 46 | 47 | class LogicalHeader(implicit p: Parameters) extends junctions.ParameterizedBundle()(p) { 48 | val src = UInt(width = p(LNHeaderBits)) 49 | val dst = UInt(width = p(LNHeaderBits)) 50 | } 51 | 52 | class LogicalNetworkIO[T <: Data](dType: T)(implicit p: Parameters) extends Bundle { 53 | val header = new LogicalHeader 54 | val payload = dType.cloneType 55 | override def cloneType = new LogicalNetworkIO(dType)(p).asInstanceOf[this.type] 56 | } 57 | 58 | object DecoupledLogicalNetworkIOWrapper { 59 | def apply[T <: Data]( 60 | in: DecoupledIO[T], 61 | src: UInt = UInt(0), 62 | dst: UInt = UInt(0)) 63 | (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { 64 | val out = Wire(Decoupled(new LogicalNetworkIO(in.bits))) 65 | out.valid := in.valid 66 | out.bits.payload := in.bits 67 | out.bits.header.dst := dst 68 | out.bits.header.src := src 69 | in.ready := out.ready 70 | out 71 | } 72 | } 73 | 74 | object DecoupledLogicalNetworkIOUnwrapper { 75 | def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) 76 | (implicit p: Parameters): DecoupledIO[T] = { 77 | val out = Wire(Decoupled(in.bits.payload)) 78 | out.valid := in.valid 79 | out.bits := in.bits.payload 80 | in.ready := out.ready 81 | out 82 | } 83 | } 84 | 85 | object DefaultFromPhysicalShim { 86 | def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]) 87 | (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { 88 | val out = Wire(Decoupled(new LogicalNetworkIO(in.bits.payload))) 89 | out.bits.header := in.bits.header 90 | out.bits.payload := in.bits.payload 91 | out.valid := in.valid 92 | in.ready := out.ready 93 | out 94 | } 95 | } 96 | 97 | object DefaultToPhysicalShim { 98 | def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]) 99 | (implicit p: Parameters): DecoupledIO[PhysicalNetworkIO[T]] = { 100 | val out = Wire(Decoupled(new PhysicalNetworkIO(n, in.bits.payload))) 101 | out.bits.header := in.bits.header 102 | out.bits.payload := in.bits.payload 103 | out.valid := in.valid 104 | in.ready := out.ready 105 | out 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/main/scala/package.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package object uncore extends uncore.constants.MemoryOpConstants 4 | { 5 | implicit def toOption[A](a: A) = Option(a) 6 | } 7 | -------------------------------------------------------------------------------- /src/main/scala/prci.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | 5 | import Chisel._ 6 | import Chisel.ImplicitConversions._ 7 | import junctions._ 8 | import junctions.NastiConstants._ 9 | import cde.{Parameters, Field} 10 | 11 | /** Number of tiles */ 12 | case object NTiles extends Field[Int] 13 | 14 | class PRCITileIO(implicit p: Parameters) extends Bundle { 15 | val reset = Bool(OUTPUT) 16 | val id = UInt(OUTPUT, log2Up(p(NTiles))) 17 | val interrupts = new Bundle { 18 | val mtip = Bool() 19 | val msip = Bool() 20 | val meip = Bool() 21 | val seip = Bool() 22 | val debug = Bool() 23 | }.asOutput 24 | 25 | override def cloneType: this.type = new PRCITileIO().asInstanceOf[this.type] 26 | } 27 | 28 | class PRCI(implicit val p: Parameters) extends Module 29 | with HasTileLinkParameters 30 | with HasAddrMapParameters { 31 | val io = new Bundle { 32 | val id = UInt(INPUT, log2Up(p(NTiles))) 33 | val interrupts = new Bundle { 34 | val mtip = Bool() 35 | val meip = Bool() 36 | val seip = Bool() 37 | val debug = Bool() 38 | }.asInput 39 | val tl = new ClientUncachedTileLinkIO().flip 40 | val tile = new PRCITileIO 41 | } 42 | 43 | val ipi = Reg(init=Bool(false)) 44 | 45 | val acq = Queue(io.tl.acquire, 1) 46 | val read = acq.bits.isBuiltInType(Acquire.getType) 47 | val write = acq.bits.isBuiltInType(Acquire.putType) 48 | val rdata = Wire(init=ipi) 49 | io.tl.grant.valid := acq.valid 50 | acq.ready := io.tl.grant.ready 51 | io.tl.grant.bits := Grant( 52 | is_builtin_type = Bool(true), 53 | g_type = acq.bits.getBuiltInGrantType(), 54 | client_xact_id = acq.bits.client_xact_id, 55 | manager_xact_id = UInt(0), 56 | addr_beat = UInt(0), 57 | data = rdata) 58 | 59 | val regSize = 16 60 | val nRegs = 2 61 | val addr = acq.bits.full_addr()(log2Up(regSize*nRegs)-1,log2Up(regSize)) 62 | 63 | when (addr === UInt(0) && write) { 64 | ipi := acq.bits.data(0) 65 | } 66 | 67 | io.tile.interrupts := io.interrupts 68 | io.tile.interrupts.msip := ipi 69 | io.tile.id := io.id 70 | } 71 | -------------------------------------------------------------------------------- /src/main/scala/rom.scala: -------------------------------------------------------------------------------- 1 | package uncore 2 | 3 | import Chisel._ 4 | import junctions._ 5 | import cde.{Parameters, Field} 6 | 7 | class ROMSlave(contents: Seq[Byte])(implicit val p: Parameters) extends Module 8 | with HasTileLinkParameters 9 | with HasAddrMapParameters { 10 | val io = new ClientUncachedTileLinkIO().flip 11 | 12 | val acq = Queue(io.acquire, 1) 13 | val single_beat = acq.bits.isBuiltInType(Acquire.getType) 14 | val multi_beat = acq.bits.isBuiltInType(Acquire.getBlockType) 15 | assert(!acq.valid || single_beat || multi_beat, "unsupported ROMSlave operation") 16 | 17 | val addr_beat = Reg(UInt()) 18 | when (io.grant.fire()) { addr_beat := addr_beat + UInt(1) } 19 | when (io.acquire.fire()) { addr_beat := io.acquire.bits.addr_beat } 20 | 21 | val byteWidth = tlDataBits / 8 22 | val rows = (contents.size + byteWidth - 1)/byteWidth + 1 23 | val rom = Vec.tabulate(rows) { i => 24 | val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) 25 | UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }, byteWidth*8) 26 | } 27 | val raddr = Cat(acq.bits.addr_block, addr_beat) 28 | val rdata = rom(if (rows == 1) UInt(0) else raddr(log2Up(rom.size)-1,0)) 29 | 30 | val last = !multi_beat || addr_beat === UInt(tlDataBeats-1) 31 | io.grant.valid := acq.valid 32 | acq.ready := io.grant.ready && last 33 | io.grant.bits := Grant( 34 | is_builtin_type = Bool(true), 35 | g_type = acq.bits.getBuiltInGrantType(), 36 | client_xact_id = acq.bits.client_xact_id, 37 | manager_xact_id = UInt(0), 38 | addr_beat = addr_beat, 39 | data = rdata) 40 | } 41 | 42 | class NastiROM(contents: Seq[Byte])(implicit p: Parameters) extends Module { 43 | val io = new NastiIO().flip 44 | val ar = Queue(io.ar, 1) 45 | 46 | // This assumes ROMs are in read-only parts of the address map. 47 | // Reuse b_queue code from NastiErrorSlave if this assumption is bad. 48 | when (ar.valid) { assert(ar.bits.len === UInt(0), "Can't burst-read from NastiROM") } 49 | assert(!(io.aw.valid || io.w.valid), "Can't write to NastiROM") 50 | io.aw.ready := Bool(false) 51 | io.w.ready := Bool(false) 52 | io.b.valid := Bool(false) 53 | 54 | val byteWidth = io.r.bits.nastiXDataBits / 8 55 | val rows = (contents.size + byteWidth - 1)/byteWidth + 1 56 | val rom = Vec.tabulate(rows) { i => 57 | val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) 58 | UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }, byteWidth*8) 59 | } 60 | val rdata_word = rom(if (rows == 1) UInt(0) else ar.bits.addr(log2Up(contents.size)-1,log2Up(byteWidth))) 61 | val rdata = new LoadGen(Cat(UInt(1), ar.bits.size), ar.bits.addr, rdata_word, Bool(false), byteWidth).data 62 | 63 | io.r <> ar 64 | io.r.bits := NastiReadDataChannel(ar.bits.id, rdata) 65 | } 66 | -------------------------------------------------------------------------------- /src/main/scala/rtc.scala: -------------------------------------------------------------------------------- 1 | package uncore 2 | 3 | import Chisel._ 4 | import junctions._ 5 | import cde.{Parameters, Field} 6 | 7 | case object RTCPeriod extends Field[Int] 8 | 9 | class RTC(nHarts: Int)(implicit val p: Parameters) extends Module 10 | with HasTileLinkParameters 11 | with HasAddrMapParameters { 12 | val io = new Bundle { 13 | val tl = new ClientUncachedTileLinkIO().flip 14 | val irqs = Vec(nHarts, Bool()).asOutput 15 | } 16 | 17 | val w = 64 18 | val regs = Reg(Vec(nHarts+1, UInt(width = w))) 19 | require(w == tlDataBits) // TODO relax this constraint for narrower TL 20 | 21 | val acq = Queue(io.tl.acquire, 1) 22 | 23 | val full_addr = acq.bits.full_addr() 24 | val byte_addr = full_addr(log2Up(w/8)-1,0) 25 | val size = w/8*(nHarts+1) 26 | val addr = full_addr(log2Up(size)-1,log2Up(w/8)) 27 | val rdata = regs(addr) 28 | val wdata = acq.bits.data 29 | val read = acq.bits.a_type === Acquire.getType 30 | val write = acq.bits.a_type === Acquire.putType 31 | val wmask = acq.bits.full_wmask() 32 | assert(!acq.valid || read || write, "unsupported RTC operation") 33 | 34 | io.tl.grant.valid := acq.valid 35 | acq.ready := io.tl.grant.ready 36 | io.tl.grant.bits := Grant( 37 | is_builtin_type = Bool(true), 38 | g_type = acq.bits.getBuiltInGrantType(), 39 | client_xact_id = acq.bits.client_xact_id, 40 | manager_xact_id = UInt(0), 41 | addr_beat = UInt(0), 42 | data = rdata) 43 | 44 | for ((irq, cmp) <- io.irqs zip regs.tail) 45 | irq := (regs(0) >= cmp) 46 | 47 | when (Counter(p(RTCPeriod)).inc()) { regs(0) := regs(0) + UInt(1) } 48 | when (acq.valid && write) { regs(addr) := wdata & wmask | rdata & ~wmask } 49 | when (reset) { regs(0) := UInt(0) } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/scala/scr.scala: -------------------------------------------------------------------------------- 1 | package uncore 2 | 3 | import Chisel._ 4 | import junctions.{SmiIO, GlobalMaps,ParameterizedBundle} 5 | import cde.{Parameters, Field} 6 | import scala.collection.mutable.HashMap 7 | 8 | case object SCRKey extends Field[SCRParameters] 9 | case class SCRParameters(nCores: Int, offsetBits: Int, csrDataBits: Int, nSCR: Int = 64) 10 | 11 | trait HasSCRParameters { 12 | implicit val p: Parameters 13 | val scrExternal = p(SCRKey) 14 | val nSCR = scrExternal.nSCR 15 | val scrAddrBits = log2Up(nSCR) 16 | val scrDataBits = 64 17 | val scrDataBytes = scrDataBits / 8 18 | val csrDataBits = scrExternal.csrDataBits 19 | val csrDataBytes = csrDataBits / 8 20 | val offsetBits = scrExternal.offsetBits 21 | val nCores = scrExternal.nCores 22 | } 23 | 24 | abstract class SCRModule(implicit val p: Parameters) extends Module with HasSCRParameters 25 | abstract class SCRBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) 26 | with HasSCRParameters 27 | 28 | /** Stores a map between SCR file names and address in the SCR file, which can 29 | * later be dumped to a header file for the test bench. */ 30 | class SCRFileMap(prefix: String, maxAddress: Int, baseAddress: BigInt) { 31 | private val addr2name = HashMap.empty[Int, String] 32 | private val name2addr = HashMap.empty[String, Int] 33 | 34 | def allocate(address: Int, name: String): Int = { 35 | Predef.assert(!addr2name.contains(address), "address already allocated") 36 | Predef.assert(!name2addr.contains(name), "name already allocated") 37 | Predef.assert(address < maxAddress, "address too large") 38 | addr2name += (address -> name) 39 | name2addr += (name -> address) 40 | println(prefix + ": %x -> ".format(baseAddress + address) + name) 41 | address 42 | } 43 | 44 | def allocate(name: String): Int = { 45 | val addr = (0 until maxAddress).filter{ addr => !addr2name.contains(addr) }(0) 46 | allocate(addr, name) 47 | } 48 | 49 | def as_c_header(): String = { 50 | addr2name.map{ case(address, name) => 51 | List( 52 | "#define " + prefix + "__" + name + "__PADDR 0x%x".format(baseAddress + address), 53 | "#define " + prefix + "__" + name + "__OFFSET 0x%x".format(address) 54 | ) 55 | }.flatten.mkString("\n") + "\n" 56 | } 57 | } 58 | 59 | class SCRIO(map: SCRFileMap)(implicit p: Parameters) extends SCRBundle()(p) { 60 | val rdata = Vec(nSCR, Bits(INPUT, scrDataBits)) 61 | val wen = Bool(OUTPUT) 62 | val waddr = UInt(OUTPUT, log2Up(nSCR)) 63 | val wdata = Bits(OUTPUT, scrDataBits) 64 | 65 | def attach(regs: Seq[Data], name_base: String): Seq[Data] = { 66 | regs.zipWithIndex.map{ case(reg, i) => attach(reg, name_base + "__" + i) } 67 | } 68 | 69 | def attach(reg: Data, name: String): Data = { 70 | val addr = map.allocate(name) 71 | when (wen && (waddr === UInt(addr))) { 72 | reg := wdata 73 | } 74 | rdata(addr) := reg 75 | reg 76 | } 77 | 78 | def allocate(address: Int, name: String): Unit = { 79 | map.allocate(address, name) 80 | } 81 | } 82 | 83 | class SCRFile(prefix: String, baseAddress: BigInt)(implicit p: Parameters) extends SCRModule()(p) { 84 | val map = new SCRFileMap(prefix, 64, baseAddress) 85 | AllSCRFiles += map 86 | 87 | val io = new Bundle { 88 | val smi = new SmiIO(scrDataBits, scrAddrBits).flip 89 | val scr = new SCRIO(map) 90 | } 91 | 92 | val scr_rdata = Wire(Vec(io.scr.rdata.size, Bits(width=scrDataBits))) 93 | for (i <- 0 until scr_rdata.size) 94 | scr_rdata(i) := io.scr.rdata(i) 95 | 96 | val read_addr = Reg(init = UInt(0, scrAddrBits)) 97 | val resp_valid = Reg(init = Bool(false)) 98 | 99 | io.smi.req.ready := !resp_valid 100 | io.smi.resp.valid := resp_valid 101 | io.smi.resp.bits := scr_rdata(read_addr) 102 | 103 | io.scr.wen := io.smi.req.fire() && io.smi.req.bits.rw 104 | io.scr.wdata := io.smi.req.bits.data 105 | io.scr.waddr := io.smi.req.bits.addr 106 | 107 | when (io.smi.req.fire()) { 108 | read_addr := io.smi.req.bits.addr 109 | resp_valid := Bool(true) 110 | } 111 | when (io.smi.resp.fire()) { resp_valid := Bool(false) } 112 | } 113 | 114 | /** Every elaborated SCR file ends up in this global arry so it can be printed 115 | * out later. */ 116 | object AllSCRFiles extends GlobalMaps[SCRFileMap] 117 | -------------------------------------------------------------------------------- /src/main/scala/tag.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | import Chisel._ 5 | import junctions._ 6 | import cde.{Parameters, Field} 7 | //import scala.collection.immutable.:: 8 | //import scala.math._ 9 | 10 | case object UseTagMem extends Field[Boolean] 11 | case object TagBits extends Field[Int] 12 | case object TagMapRatio extends Field[Int] 13 | 14 | abstract trait HasTagParameters { 15 | implicit val p: Parameters 16 | val tgBits = p(TagBits) // the number of bits in each tag 17 | val tgInstBits = 2 // size of instruction tag 18 | val tgMapRatio = p(TagMapRatio) // the number of bits a map bit represents 19 | val useTagMem = p(UseTagMem) 20 | 21 | val tgHelper = new TagUtil(tgBits, tgMapRatio, 22 | p(RAMSize), p(GlobalAddrHashMap)("mem").start, 23 | p(CacheBlockBytes)) // tag helper functions 24 | 25 | require(!useTagMem || ((tgBits >= tgInstBits * 2) && (tgBits % tgInstBits == 0))) 26 | 27 | } 28 | 29 | // support for tagged memory 30 | class TagUtil( 31 | tagBits: Int, // size of tag for each word 32 | mapRatio: Int, // tag map compression ratio 33 | memSize: BigInt, // size of tagged memory 34 | memBase: BigInt, // base address of tagged memory 35 | cacheBlockBytes: Int = 64 // byte size of a cache block 36 | ) { 37 | def wordBits = 64 // add tag to every 64-bit word 38 | def wordBytes = wordBits / 8 39 | def normTagBits = 1 << log2Up(tagBits) // normalized tag bits (around up to the nears 2's power) 40 | def tagRatio = wordBits / normTagBits // tag compression ratio 41 | def unTagBits = log2Up(tagRatio) // offset bits to get tag address 42 | def unMapBits = log2Up(mapRatio) // offset bits to get map address 43 | def tableSize = memSize / tagRatio // size of the tag table 44 | def tableBase = memBase + memSize - tableSize // base address of the tag table 45 | def map0Size = tableSize / mapRatio // size of tag map 0 46 | def map0Base = memBase + memSize - map0Size // base address of tag map 0 47 | def map1Size = map0Size / mapRatio // size of tag map 1 48 | def map1Base = memBase + memSize - map1Size // base address of tag map 1 49 | def cacheBlockTagBits = cacheBlockBytes / wordBytes * normTagBits // tag size of a cache block 50 | def cacheBlockTagBytes = cacheBlockTagBits / 8 51 | def blockOffBits = log2Up(cacheBlockBytes) 52 | def topSize = map1Size // size of the top map 53 | def topBase = map1Base // base address of the top map 54 | 55 | require(isPow2(memSize)) 56 | require(isPow2(mapRatio)) 57 | require(mapRatio >= tagRatio) // no extra space for map 58 | 59 | def tagSize(dataSize:Int) = dataSize / wordBits * tagBits 60 | def tagMaskSize(dataSize:Int) = dataSize / wordBits 61 | 62 | // convert physical address to tag table address / row byte index 63 | def pa2tta(addr: UInt): UInt = ((addr - UInt(memBase)) >> unTagBits) + UInt(tableBase) 64 | def pa2ttr(addr: UInt, rbo: Int): UInt = addr(unTagBits + rbo - 1, unTagBits) 65 | 66 | // convert physical address to tag map 0 address(a) / bit offset(b) 67 | def pa2tm0a(addr: UInt): UInt = ((addr - UInt(memBase)) >> (unTagBits + unMapBits)) + UInt(map0Base) 68 | def pa2tm0b(addr: UInt, rbo: Int): UInt = addr(unTagBits + unMapBits + rbo - 1, unTagBits + unMapBits - 3) 69 | 70 | // convert physical address to tag map 1 address(a) / bit offset(b) 71 | def pa2tm1a(addr: UInt): UInt = ((addr - UInt(memBase)) >> (unTagBits + unMapBits + unMapBits)) + UInt(map1Base) 72 | def pa2tm1b(addr: UInt, rbo: Int): UInt = addr(unTagBits + unMapBits + unMapBits + rbo - 1, unTagBits + unMapBits + unMapBits - 3) 73 | 74 | // now need to enforce writeback of tm1 and tm0 75 | def is_top(addr: UInt): Bool = (addr >> blockOffBits) >= UInt(map1Base >> blockOffBits) 76 | def is_map(addr: UInt): Bool = (addr >> blockOffBits) >= UInt(map0Base >> blockOffBits) 77 | } 78 | -------------------------------------------------------------------------------- /src/main/scala/uncore.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | import Chisel._ 5 | import cde.{Parameters, Field} 6 | 7 | case object NReleaseTransactors extends Field[Int] 8 | case object NProbeTransactors extends Field[Int] 9 | case object NAcquireTransactors extends Field[Int] 10 | 11 | /** Identifies the TLId of the inner network in a hierarchical cache controller */ 12 | case object InnerTLId extends Field[String] 13 | /** Identifies the TLId of the outer network in a hierarchical cache controller */ 14 | case object OuterTLId extends Field[String] 15 | 16 | trait HasCoherenceAgentParameters extends HasTagParameters { 17 | val nReleaseTransactors = 1 18 | val nAcquireTransactors = p(NAcquireTransactors) 19 | val nTransactors = nReleaseTransactors + nAcquireTransactors 20 | val outerTLId = p(OuterTLId) 21 | val outerTLParams = p(TLKey(outerTLId)) 22 | val outerDataBeats = outerTLParams.dataBeats 23 | val outerDataBits = outerTLParams.dataBitsPerBeat 24 | val outerBeatAddrBits = log2Up(outerDataBeats) 25 | val outerByteAddrBits = log2Up(outerDataBits/8) 26 | val outerWriteMaskBits = outerTLParams.writeMaskBits 27 | val outerTagBits = tgHelper.tagSize(outerDataBits) 28 | val outerTagMaskBits = if(useTagMem) tgHelper.tagMaskSize(outerDataBits) else 0 29 | val innerTLId = p(InnerTLId) 30 | val innerTLParams = p(TLKey(innerTLId)) 31 | val innerDataBeats = innerTLParams.dataBeats 32 | val innerDataBits = innerTLParams.dataBitsPerBeat 33 | val innerWriteMaskBits = innerTLParams.writeMaskBits 34 | val innerTagBits = tgHelper.tagSize(innerDataBits) 35 | val innerTagMaskBits = if(useTagMem) tgHelper.tagMaskSize(innerDataBits) else 0 36 | val innerBeatAddrBits = log2Up(innerDataBeats) 37 | val innerByteAddrBits = log2Up(innerDataBits/8) 38 | val maxManagerXacts = innerTLParams.maxManagerXacts 39 | require(outerDataBeats == innerDataBeats) //TODO: fix all xact_data Vecs to remove this requirement 40 | } 41 | 42 | abstract class CoherenceAgentModule(implicit val p: Parameters) extends Module 43 | with HasCoherenceAgentParameters 44 | abstract class CoherenceAgentBundle(implicit val p: Parameters) extends junctions.ParameterizedBundle()(p) 45 | with HasCoherenceAgentParameters 46 | 47 | trait HasCoherenceAgentWiringHelpers { 48 | def doOutputArbitration[T <: TileLinkChannel]( 49 | out: DecoupledIO[T], 50 | ins: Seq[DecoupledIO[T]]) { 51 | def lock(o: T) = o.hasMultibeatData() 52 | val arb = Module(new LockingRRArbiter(out.bits, ins.size, out.bits.tlDataBeats, lock _)) 53 | out <> arb.io.out 54 | arb.io.in <> ins 55 | } 56 | 57 | def doInputRouting[T <: Bundle with HasManagerTransactionId]( 58 | in: DecoupledIO[T], 59 | outs: Seq[DecoupledIO[T]]) { 60 | val idx = in.bits.manager_xact_id 61 | outs.map(_.bits := in.bits) 62 | outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } 63 | in.ready := Vec(outs.map(_.ready)).read(idx) 64 | } 65 | 66 | /** Broadcasts valid messages on this channel to all trackers, 67 | * but includes logic to allocate a new tracker in the case where 68 | * no previously allocated tracker matches the new req's addr. 69 | * 70 | * When a match is reported, if ready is high the new transaction 71 | * is merged; when ready is low the transaction is being blocked. 72 | * When no match is reported, any high readys are presumed to be 73 | * from trackers that are available for allocation, and one is 74 | * assigned via alloc based on priority; f no readys are high then 75 | * all trackers are busy with other transactions. 76 | */ 77 | def doInputRoutingWithAllocation[T <: TileLinkChannel with HasTileLinkData]( 78 | in: DecoupledIO[T], 79 | outs: Seq[DecoupledIO[T]], 80 | matches: Seq[Bool], 81 | allocs: Seq[Bool], 82 | dataOverrides: Option[Seq[UInt]] = None, 83 | allocOverride: Option[Bool] = None, 84 | matchOverride: Option[Bool] = None) { 85 | val ready_bits = Vec(outs.map(_.ready)).toBits 86 | val alloc_bits = PriorityEncoderOH(ready_bits) 87 | val match_bits = Vec(matches).toBits 88 | val no_matches = !match_bits.orR 89 | val alloc_ok = allocOverride.getOrElse(Bool(true)) 90 | val match_ok = matchOverride.getOrElse(Bool(true)) 91 | in.ready := Mux(no_matches, ready_bits.orR, (match_bits & ready_bits).orR) && alloc_ok && match_ok 92 | outs.zip(allocs).zipWithIndex.foreach { case((out, a), i) => 93 | out.valid := in.valid && match_ok 94 | out.bits := in.bits 95 | dataOverrides foreach { d => out.bits.data := d(i) } 96 | a := alloc_bits(i) & no_matches & alloc_ok 97 | } 98 | } 99 | } 100 | 101 | trait HasInnerTLIO extends HasCoherenceAgentParameters { 102 | val inner = new ManagerTileLinkIO()(p.alterPartial({case TLId => p(InnerTLId)})) 103 | val incoherent = Vec(inner.tlNCachingClients, Bool()).asInput 104 | def iacq(dummy: Int = 0) = inner.acquire.bits 105 | def iprb(dummy: Int = 0) = inner.probe.bits 106 | def irel(dummy: Int = 0) = inner.release.bits 107 | def ignt(dummy: Int = 0) = inner.grant.bits 108 | def ifin(dummy: Int = 0) = inner.finish.bits 109 | } 110 | 111 | trait HasUncachedOuterTLIO extends HasCoherenceAgentParameters { 112 | val outer = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)})) 113 | def oacq(dummy: Int = 0) = outer.acquire.bits 114 | def ognt(dummy: Int = 0) = outer.grant.bits 115 | } 116 | 117 | trait HasCachedOuterTLIO extends HasCoherenceAgentParameters { 118 | val outer = new ClientTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)})) 119 | def oacq(dummy: Int = 0) = outer.acquire.bits 120 | def oprb(dummy: Int = 0) = outer.probe.bits 121 | def orel(dummy: Int = 0) = outer.release.bits 122 | def ognt(dummy: Int = 0) = outer.grant.bits 123 | } 124 | 125 | class ManagerTLIO(implicit p: Parameters) extends CoherenceAgentBundle()(p) 126 | with HasInnerTLIO 127 | with HasUncachedOuterTLIO 128 | 129 | abstract class CoherenceAgent(implicit p: Parameters) extends CoherenceAgentModule()(p) { 130 | def innerTL: ManagerTileLinkIO 131 | def outerTL: ClientTileLinkIO 132 | def incoherent: Vec[Bool] 133 | } 134 | 135 | abstract class ManagerCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p) 136 | with HasCoherenceAgentWiringHelpers { 137 | val io = new ManagerTLIO 138 | def innerTL = io.inner 139 | lazy val outerTL = TileLinkIOWrapper(io.outer)(p.alterPartial({case TLId => p(OuterTLId)})) 140 | def incoherent = io.incoherent 141 | } 142 | 143 | class HierarchicalTLIO(implicit p: Parameters) extends CoherenceAgentBundle()(p) 144 | with HasInnerTLIO 145 | with HasCachedOuterTLIO 146 | 147 | abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p) { 148 | val io = new HierarchicalTLIO 149 | def innerTL = io.inner 150 | def outerTL = io.outer 151 | def incoherent = io.incoherent 152 | } 153 | 154 | trait HasTrackerAllocationIO extends Bundle { 155 | val matches = new Bundle { 156 | val iacq = Bool(OUTPUT) 157 | val irel = Bool(OUTPUT) 158 | val oprb = Bool(OUTPUT) 159 | } 160 | val alloc = new Bundle { 161 | val iacq = Bool(INPUT) 162 | val irel = Bool(INPUT) 163 | val oprb = Bool(INPUT) 164 | } 165 | } 166 | 167 | class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p) 168 | with HasTrackerAllocationIO 169 | 170 | class HierarchicalXactTrackerIO(implicit p: Parameters) extends HierarchicalTLIO()(p) 171 | with HasTrackerAllocationIO 172 | 173 | abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule()(p) 174 | with HasDataBeatCounters { 175 | def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = 176 | Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) 177 | 178 | def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = 179 | ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) 180 | 181 | def addPendingBitWhenId[T <: HasClientId](inc: Bool, in: T): UInt = 182 | Fill(in.tlNCachingClients, inc) & UIntToOH(in.client_id) 183 | 184 | def dropPendingBitWhenId[T <: HasClientId](dec: Bool, in: T): UInt = 185 | ~Fill(in.tlNCachingClients, dec) | ~UIntToOH(in.client_id) 186 | 187 | def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T], inc: Bool = Bool(true)): UInt = 188 | addPendingBitWhenBeat(in.fire() && in.bits.hasData() && inc, in.bits) 189 | 190 | def addPendingBitWhenBeatHasDataAndAllocs(in: DecoupledIO[AcquireFromSrc]): UInt = 191 | addPendingBitWhenBeatHasData(in, in.bits.allocate()) 192 | 193 | def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[AcquireFromSrc]): UInt = { 194 | val a = in.bits 195 | val isGetOrAtomic = a.isBuiltInType() && 196 | (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) 197 | addPendingBitWhenBeat(in.fire() && isGetOrAtomic, a) 198 | } 199 | 200 | def addPendingBitsFromAcquire(a: SecondaryMissInfo): UInt = 201 | Mux(a.hasMultibeatData(), Fill(a.tlDataBeats, UInt(1, 1)), UIntToOH(a.addr_beat)) 202 | 203 | def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = 204 | dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) 205 | 206 | def dropPendingBitAtDest[T <: HasId](in: DecoupledIO[T]): UInt = 207 | dropPendingBitWhenId(in.fire(), in.bits) 208 | 209 | def dropPendingBitAtDestWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt = 210 | dropPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits) 211 | 212 | def addPendingBitAtSrc[T <: HasId](in: DecoupledIO[T]): UInt = 213 | addPendingBitWhenId(in.fire(), in.bits) 214 | 215 | def addPendingBitAtSrcWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt = 216 | addPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits) 217 | 218 | def pinAllReadyValidLow[T <: Data](b: Bundle) { 219 | b.elements.foreach { 220 | _._2 match { 221 | case d: DecoupledIO[_] => 222 | if(d.ready.dir == OUTPUT) d.ready := Bool(false) 223 | else if(d.valid.dir == OUTPUT) d.valid := Bool(false) 224 | case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) 225 | case b: Bundle => pinAllReadyValidLow(b) 226 | case _ => 227 | } 228 | } 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /src/main/scala/util.scala: -------------------------------------------------------------------------------- 1 | // See LICENSE for license details. 2 | 3 | package uncore 4 | 5 | import Chisel._ 6 | import scala.math._ 7 | 8 | class Unsigned(x: Int) { 9 | require(x >= 0) 10 | def clog2: Int = { require(x > 0); ceil(log(x)/log(2)).toInt } 11 | def log2: Int = { require(x > 0); floor(log(x)/log(2)).toInt } 12 | def isPow2: Boolean = x > 0 && (x & (x-1)) == 0 13 | def nextPow2: Int = if (x == 0) 1 else 1 << clog2 14 | } 15 | 16 | object MuxBundle { 17 | def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { 18 | mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) 19 | } 20 | 21 | def apply[T <: Data] (key: UInt, default: T, mapping: Seq[(UInt, T)]): T = { 22 | apply(default, mapping.map{ case (a, b) => (a === key, b) }) 23 | } 24 | } 25 | 26 | // Produces 0-width value when counting to 1 27 | class ZCounter(val n: Int) { 28 | val value = Reg(init=UInt(0, log2Ceil(n))) 29 | def inc(): Bool = { 30 | if (n == 1) Bool(true) 31 | else { 32 | val wrap = value === UInt(n-1) 33 | value := Mux(Bool(!isPow2(n)) && wrap, UInt(0), value + UInt(1)) 34 | wrap 35 | } 36 | } 37 | } 38 | 39 | object ZCounter { 40 | def apply(n: Int) = new ZCounter(n) 41 | def apply(cond: Bool, n: Int): (UInt, Bool) = { 42 | val c = new ZCounter(n) 43 | var wrap: Bool = null 44 | when (cond) { wrap = c.inc() } 45 | (c.value, cond && wrap) 46 | } 47 | } 48 | 49 | object TwoWayCounter { 50 | def apply(up: Bool, down: Bool, max: Int): UInt = { 51 | val cnt = Reg(init = UInt(0, log2Up(max+1))) 52 | when (up && !down) { cnt := cnt + UInt(1) } 53 | when (down && !up) { cnt := cnt - UInt(1) } 54 | cnt 55 | } 56 | } 57 | 58 | class FlowThroughSerializer[T <: Bundle with HasTileLinkData](gen: T, n: Int) extends Module { 59 | val io = new Bundle { 60 | val in = Decoupled(gen).flip 61 | val out = Decoupled(gen) 62 | val cnt = UInt(OUTPUT, log2Up(n)) 63 | val done = Bool(OUTPUT) 64 | } 65 | val narrowWidth = io.in.bits.data.getWidth / n 66 | require(io.in.bits.data.getWidth % narrowWidth == 0) 67 | 68 | if(n == 1) { 69 | io.out <> io.in 70 | io.cnt := UInt(0) 71 | io.done := Bool(true) 72 | } else { 73 | val cnt = Reg(init=UInt(0, width = log2Up(n))) 74 | val wrap = cnt === UInt(n-1) 75 | val rbits = Reg{io.in.bits} 76 | val active = Reg(init=Bool(false)) 77 | 78 | val shifter = Wire(Vec(n, Bits(width = narrowWidth))) 79 | (0 until n).foreach { 80 | i => shifter(i) := rbits.data((i+1)*narrowWidth-1,i*narrowWidth) 81 | } 82 | 83 | io.done := Bool(false) 84 | io.cnt := cnt 85 | io.in.ready := !active 86 | io.out.valid := active || io.in.valid 87 | io.out.bits := io.in.bits 88 | when(!active && io.in.valid) { 89 | when(io.in.bits.hasData()) { 90 | cnt := Mux(io.out.ready, UInt(1), UInt(0)) 91 | rbits := io.in.bits 92 | active := Bool(true) 93 | } 94 | io.done := !io.in.bits.hasData() 95 | } 96 | when(active) { 97 | io.out.bits := rbits 98 | io.out.bits.data := shifter(cnt) 99 | when(io.out.ready) { 100 | cnt := cnt + UInt(1) 101 | when(wrap) { 102 | cnt := UInt(0) 103 | io.done := Bool(true) 104 | active := Bool(false) 105 | } 106 | } 107 | } 108 | } 109 | } 110 | 111 | object FlowThroughSerializer { 112 | def apply[T <: Bundle with HasTileLinkData](in: DecoupledIO[T], n: Int): DecoupledIO[T] = { 113 | val fs = Module(new FlowThroughSerializer(in.bits, n)) 114 | fs.io.in.valid := in.valid 115 | fs.io.in.bits := in.bits 116 | in.ready := fs.io.in.ready 117 | fs.io.out 118 | } 119 | } 120 | 121 | class SerDesBuffer(dw: Int, size: Int, ipw: Int, opw: Int) extends Module { 122 | val pointerWidth = log2Up(size) + 1 123 | val io = new Bundle { 124 | val in = Decoupled(UInt(width=ipw*dw)).flip 125 | val in_size = UInt(INPUT, width=log2Up(ipw+1)) 126 | val out = Decoupled(UInt(width=opw*dw)) 127 | val out_size = UInt(INPUT, width=log2Up(opw+1)) 128 | val count = UInt(OUTPUT, width=pointerWidth) 129 | } 130 | 131 | val wp = Reg(init = UInt(0, width=pointerWidth)) 132 | val wp_w = Wire(UInt(width=pointerWidth)) 133 | val wp_r = Wire(UInt(width=pointerWidth)) 134 | val buffer = Reg(init = UInt(0, width=size*dw)) 135 | val buffer_w = Wire(UInt(width=(size+ipw)*dw)) 136 | val din_mask = Wire(UInt(width=ipw*dw)) 137 | val buffer_r = Wire(UInt(width=size*dw)) 138 | val size_u = UInt(size) 139 | val dw_u = UInt(dw) 140 | val ipw_u = UInt(ipw) 141 | val opw_u = UInt(opw) 142 | 143 | wp_w := Mux(io.in.fire(), wp + io.in_size, wp) 144 | din_mask := FillInterleaved(dw, (UInt(1) << io.in_size) - UInt(1)) 145 | buffer_w := Mux(io.in.fire(), ((io.in.bits & din_mask) << (wp * dw_u)) | buffer, buffer) 146 | 147 | wp_r := Mux(io.out.fire(), wp_w - io.out_size, wp_w) 148 | buffer_r := buffer_w(size*dw-1,0) >> Mux(io.out.fire(), (io.out_size * dw_u), UInt(0)) 149 | 150 | wp := wp_r 151 | buffer := buffer_r 152 | 153 | io.in.ready := wp + ipw_u <= size_u 154 | io.out.valid := io.out_size =/= UInt(0) && wp >= io.out_size 155 | io.count := wp 156 | io.out.bits := buffer 157 | } 158 | -------------------------------------------------------------------------------- /src/test/verilog/tagcache/tagcache_tb.sv: -------------------------------------------------------------------------------- 1 | // Unitest test bench for tagcache 2 | 3 | `include "tagcache.consts.vh" 4 | 5 | module tb; 6 | 7 | `ifdef N_L2_TRACKERS 8 | localparam L2Xacts = `N_L2_TRACKERS; 9 | localparam TLCIS = 7; 10 | localparam TLMIS = 3; 11 | `else 12 | localparam L2Xacts = 1; 13 | localparam TLCIS = 5; 14 | localparam TLMIS = 1; 15 | `endif 16 | localparam MemAW = 32; 17 | localparam MemDW = 64; 18 | localparam MemTW = 4; 19 | localparam MemBS = 8; // burst size 20 | localparam TLAW = 32; 21 | localparam TLDW = 64; 22 | localparam TLTW = 4; 23 | localparam TLBS = 8; // burst size 24 | localparam [63:0] MemSize = `ROCKET_MEM_SIZE / MemDW * (MemDW - MemTW); 25 | localparam [63:0] MemBase = `ROCKET_MEM_BASE; 26 | 27 | //global records of transactions 28 | bit [TLAW-1:0] addr_queue[$] = {}; 29 | typedef bit [TLBS-1:0][TLDW-1:0] cache_data_t; 30 | typedef bit [TLBS-1:0][TLTW-1:0] cache_tag_t; 31 | typedef bit [TLAW-1:0] addr_t; 32 | cache_data_t memory_data_map [addr_t]; 33 | cache_tag_t memory_tag_map [addr_t]; 34 | 35 | mailbox send_queue = new(1); 36 | mailbox recv_queue = new(1); 37 | int unsigned gen_cnt = 0, chk_cnt = 0; 38 | int unsigned xact_max = 0; 39 | int unsigned client_id = 0; 40 | 41 | reg clk, reset, init; 42 | 43 | initial begin 44 | $value$plusargs("max-xact=%d", xact_max); 45 | end 46 | 47 | class TCXact; 48 | int unsigned id; 49 | rand bit [1:0] addr_type; 50 | rand int addr_offset; 51 | bit [TLAW-1:0] addr; 52 | rand bit [TLBS-1:0][TLDW-1:0] data; 53 | rand bit [TLBS-1:0][TLTW-1:0] tag; 54 | rand bit rw; 55 | rand bit burst; 56 | rand bit zero_tag; 57 | 58 | static TCXact xact_queue[$]; 59 | static int unsigned id_queue[$]; 60 | 61 | // address types: 62 | // 0: just the last one 63 | // 1: within neighbour block 64 | // 2: within the same page 65 | // 3: anywhere 66 | constraint type_constraint { addr_type dist {0 := 1, 1 := 2, 2 := 3, 3 := 3 }; } 67 | constraint offset_constraint { 68 | addr_type == 0 -> (addr_offset == 0); 69 | addr_type == 1 -> (addr_offset <= 64*1 && addr_offset >= -64*1 ); 70 | addr_type == 2 -> (addr_offset <= 64*1024 && addr_offset >= -64*1024); 71 | } 72 | constraint size_constraint { burst dist {0 := 2, 1 := 5}; } 73 | constraint tag_constraint { zero_tag dist {0 := 1, 1 := 1}; } 74 | constraint rw_constraint { rw dist {0 := 1, 1 := 2}; } 75 | 76 | function new(); 77 | endfunction // new 78 | 79 | function copy(TCXact xact); 80 | id = xact.id; 81 | addr = xact.addr; 82 | data = xact.data; 83 | tag = xact.tag; 84 | rw = xact.rw; 85 | burst = xact.burst; 86 | endfunction // copy 87 | 88 | function string toString(int resp = 0); 89 | string operation_str = rw ? "write" : "read"; 90 | string size_str = burst ? "a block" : "a beat"; 91 | int index = addr[5:0]/(TLDW/8); 92 | string beat_data = $sformatf("\n%2d: %16h,%1h", index, data[index], tag[index]); 93 | string data_str, block_data; 94 | int i; 95 | for(i=0; i 0) 112 | addr = addr_queue[0] - MemBase + addr_offset; 113 | else 114 | addr = MemSize / 2 + addr_offset; 115 | addr = addr % MemSize; 116 | addr = addr < 0 ? addr + MemSize : addr; 117 | addr = burst ? addr / 64 * 64 : addr / (TLDW/8) * (TLDW/8); 118 | addr = addr + MemBase; 119 | if(!memory_data_map.exists(addr / 64 * 64)) begin 120 | burst = 1; 121 | addr = addr / 64 * 64; 122 | end 123 | if(zero_tag) tag = 0; 124 | end else begin // read 125 | int unsigned index; 126 | addr_offset = addr_offset < 0 ? -addr_offset : addr_offset; 127 | index = addr_offset / 64 % addr_queue.size; 128 | addr = burst ? addr_queue[index] : addr_queue[index] + (addr_offset % 64) / (TLDW/8) * (TLDW/8); 129 | end // else: !if(rw == 1) 130 | 131 | id = client_id; 132 | client_id = id == 2**TLCIS - 1 ? 0 : id + 1; 133 | endfunction // post_randomize 134 | 135 | function void record(); 136 | bit [TLAW-1:0] baddr = addr / 64 * 64; 137 | int unsigned index = addr[5:0]/(TLDW/8); 138 | 139 | if(rw) addr_queue.push_front(baddr); 140 | if(addr_queue.size > 1024) addr_queue.pop_back(); 141 | 142 | xact_queue.push_back(this); 143 | id_queue.push_back(gen_cnt); 144 | endfunction 145 | 146 | function int unsigned check(); 147 | TCXact orig_xact; 148 | bit [TLAW-1:0] baddr; 149 | int unsigned index; 150 | int qi[$] = xact_queue.find_first_index(x) with (x.id == id); 151 | automatic int unsigned cnt_id; 152 | if(qi.size == 0) 153 | $fatal(1, "Get a response to an unknown transaction!n"); 154 | orig_xact = xact_queue[qi[0]]; 155 | cnt_id = id_queue[qi[0]]; 156 | addr = orig_xact.addr; 157 | burst = orig_xact.burst; 158 | baddr = addr / 64 * 64; 159 | index = addr[5:0]/(TLDW/8); 160 | if(rw) begin // write 161 | // update the data 162 | if(!memory_data_map.exists(baddr)) begin 163 | memory_data_map[baddr] = 0; 164 | memory_tag_map[baddr] = 0; 165 | end 166 | if(burst) begin 167 | memory_data_map[baddr] = orig_xact.data; 168 | memory_tag_map[baddr] = orig_xact.tag; 169 | end else begin 170 | memory_data_map[baddr][index] = orig_xact.data[index]; 171 | memory_tag_map[baddr][index] = orig_xact.tag[index]; 172 | end 173 | end else begin // read 174 | if(!memory_data_map.exists(baddr)) 175 | $fatal(1, "Read response miss in memory map!\n(%0d): %s\n", id_queue[qi[0]], toString(1)); 176 | if(burst && (memory_data_map[baddr] != data || memory_tag_map[baddr] != tag)) 177 | $fatal(1, "Read response mismatch with memory map!\n(%0d): %s\n", cnt_id, toString(1)); 178 | if(!burst && (memory_data_map[baddr][index] != data[index] || memory_tag_map[baddr][index] != tag[index])) 179 | $fatal(1, "Read response mismatch with memory map!\n(%0d): %s\n", cnt_id, toString(1)); 180 | end // else: !if(rw) 181 | xact_queue.delete(qi[0]); 182 | id_queue.delete(qi[0]); 183 | return cnt_id; 184 | endfunction // check 185 | 186 | endclass 187 | 188 | task xact_gen(); 189 | TCXact xact; 190 | while(xact_max == 0 || gen_cnt < xact_max) begin 191 | while(xact.id_queue.size() != 0 && client_id == xact.id_queue[0]) @(posedge clk); 192 | xact = new; 193 | xact.randomize(); 194 | send_queue.put(xact); 195 | xact.record(); 196 | $info("Generate a (%0d) %s\n", gen_cnt, xact.toString()); 197 | gen_cnt = gen_cnt + 1; 198 | end 199 | endtask // xact_gen 200 | 201 | task xact_check(); 202 | TCXact xact; 203 | automatic int unsigned id; 204 | while(xact_max == 0 || chk_cnt < xact_max) begin 205 | recv_queue.get(xact); 206 | id = xact.check(); 207 | chk_cnt = chk_cnt + 1; 208 | $info("Recieve a (%0d) %s\n", id, xact.toString(1)); 209 | end 210 | 211 | if(xact.xact_queue.size() != 0) 212 | $fatal(1, "Simulation finishes with more responses received than requests generated!\n",); 213 | 214 | $info("Simulation finishes OK with %d requests sent and checked.", chk_cnt); 215 | $finish(); 216 | endtask // xact_check 217 | 218 | 219 | logic io_in_acquire_ready; 220 | reg io_in_acquire_valid = 'b0; 221 | reg [TLAW-7:0] io_in_acquire_bits_addr_block; 222 | reg [TLCIS-1:0] io_in_acquire_bits_client_xact_id; 223 | reg [2:0] io_in_acquire_bits_addr_beat; 224 | reg io_in_acquire_bits_is_builtin_type; 225 | reg [2:0] io_in_acquire_bits_a_type; 226 | reg [12:0] io_in_acquire_bits_union; 227 | reg [TLDW-1:0] io_in_acquire_bits_data; 228 | reg [TLTW-1:0] io_in_acquire_bits_tag; 229 | reg io_in_acquire_bits_client_id; 230 | reg io_in_grant_ready; 231 | logic io_in_grant_valid; 232 | logic [2:0] io_in_grant_bits_addr_beat; 233 | logic [TLCIS-1:0]io_in_grant_bits_client_xact_id; 234 | logic [TLMIS-1:0]io_in_grant_bits_manager_xact_id; 235 | logic io_in_grant_bits_is_builtin_type; 236 | logic [3:0] io_in_grant_bits_g_type; 237 | logic [TLDW-1:0] io_in_grant_bits_data; 238 | logic [TLTW-1:0] io_in_grant_bits_tag; 239 | logic io_in_grant_bits_client_id; 240 | logic io_in_finish_ready; 241 | reg io_in_finish_valid = 'b0; 242 | reg [TLMIS-1:0] io_in_finish_bits_manager_xact_id; 243 | reg io_in_probe_ready; 244 | logic io_in_probe_valid; 245 | logic [TLAW-7:0] io_in_probe_bits_addr_block; 246 | logic io_in_probe_bits_p_type; 247 | logic io_in_probe_bits_client_id; 248 | logic io_in_release_ready; 249 | reg io_in_release_valid = 'b0; 250 | reg [2:0] io_in_release_bits_addr_beat; 251 | reg [TLAW-7:0] io_in_release_bits_addr_block; 252 | reg [TLCIS-1:0] io_in_release_bits_client_xact_id; 253 | reg io_in_release_bits_voluntary; 254 | reg [1:0] io_in_release_bits_r_type; 255 | reg [TLDW-1:0] io_in_release_bits_data; 256 | reg [TLTW-1:0] io_in_release_bits_tag; 257 | reg io_in_release_bits_client_id; 258 | 259 | initial begin 260 | reset = 'b1; 261 | init = 'b1; 262 | #77; 263 | reset = 0; 264 | end 265 | 266 | initial begin 267 | clk = 0; 268 | forever #5 clk = !clk; 269 | end 270 | 271 | nasti_channel 272 | #( 273 | .ID_WIDTH ( 8 ), 274 | .ADDR_WIDTH ( MemAW ), 275 | .DATA_WIDTH ( MemDW )) 276 | mem_nasti(); 277 | 278 | nasti_ram_behav 279 | #( 280 | .ID_WIDTH ( 8 ), 281 | .ADDR_WIDTH ( MemAW ), 282 | .DATA_WIDTH ( MemDW ), 283 | .USER_WIDTH ( 1 ) 284 | ) 285 | ram_behav 286 | ( 287 | .clk ( clk ), 288 | .rstn ( !reset ), 289 | .nasti ( mem_nasti ) 290 | ); 291 | 292 | task xact_send(); 293 | TCXact xact = new; 294 | while(1) begin 295 | @(posedge clk); #1; 296 | io_in_acquire_valid = 'b0; 297 | send_queue.get(xact); 298 | 299 | if(xact.rw && xact.burst) begin // write a burst 300 | int i; 301 | for(i=0; i