1 /* 2 * Copyright (c) 2019, Intel Corporation. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7 #include <arch.h> 8 #include <arch_helpers.h> 9 #include <errno.h> 10 #include <lib/mmio.h> 11 #include <lib/utils.h> 12 #include <common/debug.h> 13 #include <drivers/delay_timer.h> 14 #include <platform_def.h> 15 #include <string.h> 16 17 #include "s10_memory_controller.h" 18 #include "socfpga_reset_manager.h" 19 20 #define ALT_CCU_NOC_DI_SET_MSK 0x10 21 22 #define DDR_READ_LATENCY_DELAY 40 23 #define MAX_MEM_CAL_RETRY 3 24 #define PRE_CALIBRATION_DELAY 1 25 #define POST_CALIBRATION_DELAY 1 26 #define TIMEOUT_EMIF_CALIBRATION 1000 27 #define CLEAR_EMIF_DELAY 1000 28 #define CLEAR_EMIF_TIMEOUT 1000 29 30 #define DDR_CONFIG(A, B, C, R) (((A) << 24) | ((B) << 16) | ((C) << 8) | (R)) 31 #define DDR_CONFIG_ELEMENTS (sizeof(ddr_config)/sizeof(uint32_t)) 32 33 /* tWR = Min. 15ns constant, see JEDEC standard eg. DDR4 is JESD79-4.pdf */ 34 #define tWR_IN_NS 15 35 36 void configure_hmc_adaptor_regs(void); 37 void configure_ddr_sched_ctrl_regs(void); 38 39 /* The followring are the supported configurations */ 40 uint32_t ddr_config[] = { 41 /* DDR_CONFIG(Address order,Bank,Column,Row) */ 42 /* List for DDR3 or LPDDR3 (pinout order > chip, row, bank, column) */ 43 DDR_CONFIG(0, 3, 10, 12), 44 DDR_CONFIG(0, 3, 9, 13), 45 DDR_CONFIG(0, 3, 10, 13), 46 DDR_CONFIG(0, 3, 9, 14), 47 DDR_CONFIG(0, 3, 10, 14), 48 DDR_CONFIG(0, 3, 10, 15), 49 DDR_CONFIG(0, 3, 11, 14), 50 DDR_CONFIG(0, 3, 11, 15), 51 DDR_CONFIG(0, 3, 10, 16), 52 DDR_CONFIG(0, 3, 11, 16), 53 DDR_CONFIG(0, 3, 12, 15), /* 0xa */ 54 /* List for DDR4 only (pinout order > chip, bank, row, column) */ 55 DDR_CONFIG(1, 3, 10, 14), 56 DDR_CONFIG(1, 4, 10, 14), 57 DDR_CONFIG(1, 3, 10, 15), 58 DDR_CONFIG(1, 4, 10, 15), 59 DDR_CONFIG(1, 3, 10, 16), 60 DDR_CONFIG(1, 4, 10, 16), 61 DDR_CONFIG(1, 3, 10, 17), 62 DDR_CONFIG(1, 4, 10, 17), 63 }; 64 65 static int match_ddr_conf(uint32_t ddr_conf) 66 { 67 int i; 68 69 for (i = 0; i < DDR_CONFIG_ELEMENTS; i++) { 70 if (ddr_conf == ddr_config[i]) 71 return i; 72 } 73 return 0; 74 } 75 76 static int check_hmc_clk(void) 77 { 78 unsigned long timeout = 0; 79 uint32_t hmc_clk; 80 81 do { 82 hmc_clk = mmio_read_32(S10_SYSMGR_CORE_HMC_CLK); 83 if (hmc_clk & S10_SYSMGR_CORE_HMC_CLK_STATUS) 84 break; 85 udelay(1); 86 } while (++timeout < 1000); 87 if (timeout >= 1000) 88 return -ETIMEDOUT; 89 90 return 0; 91 } 92 93 static int clear_emif(void) 94 { 95 uint32_t data; 96 unsigned long timeout; 97 98 mmio_write_32(S10_MPFE_HMC_ADP_RSTHANDSHAKECTRL, 0); 99 100 timeout = 0; 101 do { 102 data = mmio_read_32(S10_MPFE_HMC_ADP_RSTHANDSHAKESTAT); 103 if ((data & S10_MPFE_HMC_ADP_RSTHANDSHAKESTAT_SEQ2CORE) == 0) 104 break; 105 udelay(CLEAR_EMIF_DELAY); 106 } while (++timeout < CLEAR_EMIF_TIMEOUT); 107 if (timeout >= CLEAR_EMIF_TIMEOUT) 108 return -ETIMEDOUT; 109 110 return 0; 111 } 112 113 static int mem_calibration(void) 114 { 115 int status = 0; 116 uint32_t data; 117 unsigned long timeout; 118 unsigned long retry = 0; 119 120 udelay(PRE_CALIBRATION_DELAY); 121 122 do { 123 if (retry != 0) 124 INFO("DDR: Retrying DRAM calibration\n"); 125 126 timeout = 0; 127 do { 128 data = mmio_read_32(S10_MPFE_HMC_ADP_DDRCALSTAT); 129 if (S10_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 1) 130 break; 131 udelay(500); 132 } while (++timeout < TIMEOUT_EMIF_CALIBRATION); 133 134 if (S10_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 0) { 135 status = clear_emif(); 136 if (status) 137 ERROR("Failed to clear Emif\n"); 138 } else { 139 break; 140 } 141 } while (++retry < MAX_MEM_CAL_RETRY); 142 143 if (S10_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 0) { 144 ERROR("DDR: DRAM calibration failed.\n"); 145 status = -EIO; 146 } else { 147 INFO("DDR: DRAM calibration success.\n"); 148 status = 0; 149 } 150 151 udelay(POST_CALIBRATION_DELAY); 152 153 return status; 154 } 155 156 int init_hard_memory_controller(void) 157 { 158 int status; 159 160 mmio_clrbits_32(S10_CCU_CPU0_MPRT_DDR, S10_CCU_NOC_DI_SET_MSK); 161 mmio_clrbits_32(S10_CCU_CPU0_MPRT_MEM0, S10_CCU_NOC_DI_SET_MSK); 162 mmio_clrbits_32(S10_CCU_CPU0_MPRT_MEM1A, S10_CCU_NOC_DI_SET_MSK); 163 mmio_clrbits_32(S10_CCU_CPU0_MPRT_MEM1B, S10_CCU_NOC_DI_SET_MSK); 164 mmio_clrbits_32(S10_CCU_CPU0_MPRT_MEM1C, S10_CCU_NOC_DI_SET_MSK); 165 mmio_clrbits_32(S10_CCU_CPU0_MPRT_MEM1D, S10_CCU_NOC_DI_SET_MSK); 166 mmio_clrbits_32(S10_CCU_CPU0_MPRT_MEM1E, S10_CCU_NOC_DI_SET_MSK); 167 168 mmio_clrbits_32(S10_CCU_IOM_MPRT_MEM0, S10_CCU_NOC_DI_SET_MSK); 169 mmio_clrbits_32(S10_CCU_IOM_MPRT_MEM1A, S10_CCU_NOC_DI_SET_MSK); 170 mmio_clrbits_32(S10_CCU_IOM_MPRT_MEM1B, S10_CCU_NOC_DI_SET_MSK); 171 mmio_clrbits_32(S10_CCU_IOM_MPRT_MEM1C, S10_CCU_NOC_DI_SET_MSK); 172 mmio_clrbits_32(S10_CCU_IOM_MPRT_MEM1D, S10_CCU_NOC_DI_SET_MSK); 173 mmio_clrbits_32(S10_CCU_IOM_MPRT_MEM1E, S10_CCU_NOC_DI_SET_MSK); 174 175 mmio_write_32(S10_NOC_FW_DDR_SCR_MPUREGION0ADDR_LIMIT, 0xFFFF0000); 176 mmio_write_32(S10_NOC_FW_DDR_SCR_MPUREGION0ADDR_LIMITEXT, 0x1F); 177 178 mmio_write_32(S10_NOC_FW_DDR_SCR_NONMPUREGION0ADDR_LIMIT, 0xFFFF0000); 179 mmio_write_32(S10_NOC_FW_DDR_SCR_NONMPUREGION0ADDR_LIMITEXT, 0x1F); 180 mmio_write_32(S10_SOC_NOC_FW_DDR_SCR_ENABLE, BIT(0) | BIT(8)); 181 182 status = check_hmc_clk(); 183 if (status) { 184 ERROR("DDR: Error, HMC clock not running\n"); 185 return status; 186 } 187 188 mmio_clrbits_32(SOCFPGA_RSTMGR(BRGMODRST), RSTMGR_FIELD(BRG, DDRSCH)); 189 190 status = mem_calibration(); 191 if (status) { 192 ERROR("DDR: Memory Calibration Failed\n"); 193 return status; 194 } 195 196 configure_hmc_adaptor_regs(); 197 configure_ddr_sched_ctrl_regs(); 198 199 return 0; 200 } 201 202 void configure_ddr_sched_ctrl_regs(void) 203 { 204 uint32_t data, dram_addr_order, ddr_conf, bank, row, col, 205 rd_to_miss, wr_to_miss, burst_len, burst_len_ddr_clk, 206 burst_len_sched_clk, act_to_act, rd_to_wr, wr_to_rd, bw_ratio, 207 t_rtp, t_rp, t_rcd, rd_latency, tw_rin_clk_cycles, 208 bw_ratio_extended, auto_precharge = 0, act_to_act_bank, faw, 209 faw_bank, bus_rd_to_rd, bus_rd_to_wr, bus_wr_to_rd; 210 211 INFO("Init HPS NOC's DDR Scheduler.\n"); 212 213 data = mmio_read_32(S10_MPFE_IOHMC_CTRLCFG1); 214 dram_addr_order = S10_MPFE_IOHMC_CTRLCFG1_CFG_ADDR_ORDER(data); 215 216 data = mmio_read_32(S10_MPFE_IOHMC_DRAMADDRW); 217 218 col = IOHMC_DRAMADDRW_COL_ADDR_WIDTH(data); 219 row = IOHMC_DRAMADDRW_ROW_ADDR_WIDTH(data); 220 bank = IOHMC_DRAMADDRW_BANK_ADDR_WIDTH(data) + 221 IOHMC_DRAMADDRW_BANK_GRP_ADDR_WIDTH(data); 222 223 ddr_conf = match_ddr_conf(DDR_CONFIG(dram_addr_order, bank, col, row)); 224 225 if (ddr_conf) { 226 mmio_clrsetbits_32( 227 S10_MPFE_DDR_MAIN_SCHED_DDRCONF, 228 S10_MPFE_DDR_MAIN_SCHED_DDRCONF_SET_MSK, 229 S10_MPFE_DDR_MAIN_SCHED_DDRCONF_SET(ddr_conf)); 230 } else { 231 ERROR("DDR: Cannot find predefined ddrConf configuration.\n"); 232 } 233 234 mmio_write_32(S10_MPFE_HMC_ADP(ADP_DRAMADDRWIDTH), data); 235 236 data = mmio_read_32(S10_MPFE_IOHMC_DRAMTIMING0); 237 rd_latency = S10_MPFE_IOHMC_REG_DRAMTIMING0_CFG_TCL(data); 238 239 data = mmio_read_32(S10_MPFE_IOHMC_CALTIMING0); 240 act_to_act = ACT_TO_ACT(data); 241 t_rcd = ACT_TO_RDWR(data); 242 act_to_act_bank = ACT_TO_ACT_DIFF_BANK(data); 243 244 data = mmio_read_32(S10_MPFE_IOHMC_CALTIMING1); 245 rd_to_wr = RD_TO_WR(data); 246 bus_rd_to_rd = RD_TO_RD_DIFF_CHIP(data); 247 bus_rd_to_wr = RD_TO_WR_DIFF_CHIP(data); 248 249 data = mmio_read_32(S10_MPFE_IOHMC_CALTIMING2); 250 t_rtp = RD_TO_PCH(data); 251 252 data = mmio_read_32(S10_MPFE_IOHMC_CALTIMING3); 253 wr_to_rd = CALTIMING3_WR_TO_RD(data); 254 bus_wr_to_rd = CALTIMING3_WR_TO_RD_DIFF_CHIP(data); 255 256 data = mmio_read_32(S10_MPFE_IOHMC_CALTIMING4); 257 t_rp = PCH_TO_VALID(data); 258 259 data = mmio_read_32(S10_MPFE_HMC_ADP(HMC_ADP_DDRIOCTRL)); 260 bw_ratio = ((HMC_ADP_DDRIOCTRL_IO_SIZE(data) == 0) ? 0 : 1); 261 262 data = mmio_read_32(S10_MPFE_IOHMC_CTRLCFG0); 263 burst_len = HMC_ADP_DDRIOCTRL_CTRL_BURST_LENGTH(data); 264 burst_len_ddr_clk = burst_len / 2; 265 burst_len_sched_clk = ((burst_len/2) / 2); 266 267 data = mmio_read_32(S10_MPFE_IOHMC_CTRLCFG0); 268 switch (S10_MPFE_IOHMC_REG_CTRLCFG0_CFG_MEM_TYPE(data)) { 269 case 1: 270 /* DDR4 - 1333MHz */ 271 /* 20 (19.995) clock cycles = 15ns */ 272 /* Calculate with rounding */ 273 tw_rin_clk_cycles = (((tWR_IN_NS * 1333) % 1000) >= 500) ? 274 ((tWR_IN_NS * 1333) / 1000) + 1 : 275 ((tWR_IN_NS * 1333) / 1000); 276 break; 277 default: 278 /* Others - 1066MHz or slower */ 279 /* 16 (15.990) clock cycles = 15ns */ 280 /* Calculate with rounding */ 281 tw_rin_clk_cycles = (((tWR_IN_NS * 1066) % 1000) >= 500) ? 282 ((tWR_IN_NS * 1066) / 1000) + 1 : 283 ((tWR_IN_NS * 1066) / 1000); 284 break; 285 } 286 287 rd_to_miss = t_rtp + t_rp + t_rcd - burst_len_sched_clk; 288 wr_to_miss = ((rd_latency + burst_len_ddr_clk + 2 + tw_rin_clk_cycles) 289 / 2) - rd_to_wr + t_rp + t_rcd; 290 291 mmio_write_32(S10_MPFE_DDR_MAIN_SCHED_DDRTIMING, 292 bw_ratio << DDRTIMING_BWRATIO_OFST | 293 wr_to_rd << DDRTIMING_WRTORD_OFST| 294 rd_to_wr << DDRTIMING_RDTOWR_OFST | 295 burst_len_sched_clk << DDRTIMING_BURSTLEN_OFST | 296 wr_to_miss << DDRTIMING_WRTOMISS_OFST | 297 rd_to_miss << DDRTIMING_RDTOMISS_OFST | 298 act_to_act << DDRTIMING_ACTTOACT_OFST); 299 300 data = mmio_read_32(S10_MPFE_HMC_ADP(HMC_ADP_DDRIOCTRL)); 301 bw_ratio_extended = ((ADP_DDRIOCTRL_IO_SIZE(data) == 0) ? 1 : 0); 302 303 mmio_write_32(S10_MPFE_DDR_MAIN_SCHED_DDRMODE, 304 bw_ratio_extended << DDRMODE_BWRATIOEXTENDED_OFST | 305 auto_precharge << DDRMODE_AUTOPRECHARGE_OFST); 306 307 mmio_write_32(S10_MPFE_DDR_MAIN_SCHED_READLATENCY, 308 (rd_latency / 2) + DDR_READ_LATENCY_DELAY); 309 310 data = mmio_read_32(S10_MPFE_IOHMC_CALTIMING9); 311 faw = S10_MPFE_IOHMC_CALTIMING9_ACT_TO_ACT(data); 312 313 faw_bank = 1; // always 1 because we always have 4 bank DDR. 314 315 mmio_write_32(S10_MPFE_DDR_MAIN_SCHED_ACTIVATE, 316 faw_bank << S10_MPFE_DDR_MAIN_SCHED_ACTIVATE_FAWBANK_OFST | 317 faw << S10_MPFE_DDR_MAIN_SCHED_ACTIVATE_FAW_OFST | 318 act_to_act_bank << S10_MPFE_DDR_MAIN_SCHED_ACTIVATE_RRD_OFST); 319 320 mmio_write_32(S10_MPFE_DDR_MAIN_SCHED_DEVTODEV, 321 ((bus_rd_to_rd 322 << S10_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTORD_OFST) 323 & S10_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTORD_MSK) | 324 ((bus_rd_to_wr 325 << S10_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTOWR_OFST) 326 & S10_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTOWR_MSK) | 327 ((bus_wr_to_rd 328 << S10_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSWRTORD_OFST) 329 & S10_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSWRTORD_MSK)); 330 331 } 332 333 unsigned long get_physical_dram_size(void) 334 { 335 uint32_t data; 336 unsigned long ram_addr_width, ram_ext_if_io_width; 337 338 data = mmio_read_32(S10_MPFE_HMC_ADP_DDRIOCTRL); 339 switch (S10_MPFE_HMC_ADP_DDRIOCTRL_IO_SIZE(data)) { 340 case 0: 341 ram_ext_if_io_width = 16; 342 break; 343 case 1: 344 ram_ext_if_io_width = 32; 345 break; 346 case 2: 347 ram_ext_if_io_width = 64; 348 break; 349 default: 350 ram_ext_if_io_width = 0; 351 break; 352 } 353 354 data = mmio_read_32(S10_MPFE_IOHMC_REG_DRAMADDRW); 355 ram_addr_width = IOHMC_DRAMADDRW_CFG_COL_ADDR_WIDTH(data) + 356 IOHMC_DRAMADDRW_CFG_ROW_ADDR_WIDTH(data) + 357 IOHMC_DRAMADDRW_CFG_BANK_ADDR_WIDTH(data) + 358 IOHMC_DRAMADDRW_CFG_BANK_GROUP_ADDR_WIDTH(data) + 359 IOHMC_DRAMADDRW_CFG_CS_ADDR_WIDTH(data); 360 361 return (1 << ram_addr_width) * (ram_ext_if_io_width / 8); 362 } 363 364 365 366 void configure_hmc_adaptor_regs(void) 367 { 368 uint32_t data; 369 uint32_t dram_io_width; 370 371 dram_io_width = S10_MPFE_IOHMC_NIOSRESERVE0_NIOS_RESERVE0( 372 mmio_read_32(S10_MPFE_IOHMC_REG_NIOSRESERVE0_OFST)); 373 374 dram_io_width = (dram_io_width & 0xFF) >> 5; 375 376 mmio_clrsetbits_32(S10_MPFE_HMC_ADP_DDRIOCTRL, 377 S10_MPFE_HMC_ADP_DDRIOCTRL_IO_SIZE_MSK, 378 dram_io_width << S10_MPFE_HMC_ADP_DDRIOCTRL_IO_SIZE_OFST); 379 380 mmio_write_32(S10_MPFE_HMC_ADP_HPSINTFCSEL, 381 S10_MPFE_HMC_ADP_HPSINTFCSEL_ENABLE); 382 383 data = mmio_read_32(S10_MPFE_IOHMC_REG_CTRLCFG1); 384 if (data & (1 << S10_IOHMC_CTRLCFG1_ENABLE_ECC_OFST)) { 385 mmio_clrsetbits_32(S10_MPFE_HMC_ADP_ECCCTRL1, 386 S10_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK | 387 S10_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK | 388 S10_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK, 389 S10_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK | 390 S10_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK); 391 392 mmio_clrsetbits_32(S10_MPFE_HMC_ADP_ECCCTRL2, 393 S10_MPFE_HMC_ADP_ECCCTRL2_OVRW_RB_ECC_EN_SET_MSK | 394 S10_MPFE_HMC_ADP_ECCCTRL2_RMW_EN_SET_MSK | 395 S10_MPFE_HMC_ADP_ECCCTRL2_AUTOWB_EN_SET_MSK, 396 S10_MPFE_HMC_ADP_ECCCTRL2_RMW_EN_SET_MSK | 397 S10_MPFE_HMC_ADP_ECCCTRL2_AUTOWB_EN_SET_MSK); 398 399 mmio_clrsetbits_32(S10_MPFE_HMC_ADP_ECCCTRL1, 400 S10_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK | 401 S10_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK | 402 S10_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK, 403 S10_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK); 404 INFO("Scrubbing ECC\n"); 405 406 /* ECC Scrubbing */ 407 zeromem(DRAM_BASE, DRAM_SIZE); 408 } else { 409 INFO("ECC is disabled.\n"); 410 } 411 } 412 413