AMC13
Firmwares for the different applications of the AMC13 uTCA board made at Boston University
 All Classes Variables
mig_7series_v1_9_ddr_phy_rdlvl.v
1  //*****************************************************************************
2 // (c) Copyright 2009 - 2013 Xilinx, Inc. All rights reserved.
3 //
4 // This file contains confidential and proprietary information
5 // of Xilinx, Inc. and is protected under U.S. and
6 // international copyright and other intellectual property
7 // laws.
8 //
9 // DISCLAIMER
10 // This disclaimer is not a license and does not grant any
11 // rights to the materials distributed herewith. Except as
12 // otherwise provided in a valid license issued to you by
13 // Xilinx, and to the maximum extent permitted by applicable
14 // law: (1) THESE MATERIALS ARE MADE AVAILABLE "AS IS" AND
15 // WITH ALL FAULTS, AND XILINX HEREBY DISCLAIMS ALL WARRANTIES
16 // AND CONDITIONS, EXPRESS, IMPLIED, OR STATUTORY, INCLUDING
17 // BUT NOT LIMITED TO WARRANTIES OF MERCHANTABILITY, NON-
18 // INFRINGEMENT, OR FITNESS FOR ANY PARTICULAR PURPOSE; and
19 // (2) Xilinx shall not be liable (whether in contract or tort,
20 // including negligence, or under any other theory of
21 // liability) for any loss or damage of any kind or nature
22 // related to, arising under or in connection with these
23 // materials, including for any direct, or any indirect,
24 // special, incidental, or consequential loss or damage
25 // (including loss of data, profits, goodwill, or any type of
26 // loss or damage suffered as a result of any action brought
27 // by a third party) even if such damage or loss was
28 // reasonably foreseeable or Xilinx had been advised of the
29 // possibility of the same.
30 //
31 // CRITICAL APPLICATIONS
32 // Xilinx products are not designed or intended to be fail-
33 // safe, or for use in any application requiring fail-safe
34 // performance, such as life-support or safety devices or
35 // systems, Class III medical devices, nuclear facilities,
36 // applications related to the deployment of airbags, or any
37 // other applications that could lead to death, personal
38 // injury, or severe property or environmental damage
39 // (individually and collectively, "Critical
40 // Applications"). Customer assumes the sole risk and
41 // liability of any use of Xilinx products in Critical
42 // Applications, subject only to applicable laws and
43 // regulations governing limitations on product liability.
44 //
45 // THIS COPYRIGHT NOTICE AND DISCLAIMER MUST BE RETAINED AS
46 // PART OF THIS FILE AT ALL TIMES.
47 //
48 //*****************************************************************************
49 // ____ ____
50 // / /\/ /
51 // /___/ \ / Vendor: Xilinx
52 // \ \ \/ Version:
53 // \ \ Application: MIG
54 // / / Filename: ddr_phy_rdlvl.v
55 // /___/ /\ Date Last Modified: $Date: 2011/06/24 14:49:00 $
56 // \ \ / \ Date Created:
57 // \___\/\___\
58 //
59 //Device: 7 Series
60 //Design Name: DDR3 SDRAM
61 //Purpose:
62 // Read leveling Stage1 calibration logic
63 // NOTES:
64 // 1. Window detection with PRBS pattern.
65 //Reference:
66 //Revision History:
67 //*****************************************************************************
68 
69 /******************************************************************************
70 **$Id: ddr_phy_rdlvl.v,v 1.2 2011/06/24 14:49:00 mgeorge Exp $
71 **$Date: 2011/06/24 14:49:00 $
72 **$Author: mgeorge $
73 **$Revision: 1.2 $
74 **$Source: /devl/xcs/repo/env/Databases/ip/src2/O/mig_7series_v1_3/data/dlib/7series/ddr3_sdram/verilog/rtl/phy/ddr_phy_rdlvl.v,v $
75 *******************************************************************************/
76 
77 `timescale 1ps/1ps
78 
79 (* use_dsp48 = "no" *)
80 
81 module mig_7series_v1_9_ddr_phy_rdlvl #
82  (
83  parameter TCQ = 100, // clk->out delay (sim only)
84  parameter nCK_PER_CLK = 2, // # of memory clocks per CLK
85  parameter CLK_PERIOD = 3333, // Internal clock period (in ps)
86  parameter DQ_WIDTH = 64, // # of DQ (data)
87  parameter DQS_CNT_WIDTH = 3, // = ceil(log2(DQS_WIDTH))
88  parameter DQS_WIDTH = 8, // # of DQS (strobe)
89  parameter DRAM_WIDTH = 8, // # of DQ per DQS
90  parameter RANKS = 1, // # of DRAM ranks
91  parameter PER_BIT_DESKEW = "ON", // Enable per-bit DQ deskew
92  parameter SIM_CAL_OPTION = "NONE", // Skip various calibration steps
93  parameter DEBUG_PORT = "OFF", // Enable debug port
94  parameter DRAM_TYPE = "DDR3", // Memory I/F type: "DDR3", "DDR2"
95  parameter OCAL_EN = "ON"
96  )
97  (
98  input clk,
99  input rst,
100  // Calibration status, control signals
101  input mpr_rdlvl_start,
102  output mpr_rdlvl_done,
103  output reg mpr_last_byte_done,
104  output mpr_rnk_done,
105  input rdlvl_stg1_start,
106 (* keep = "true", max_fanout = 30 *) output reg rdlvl_stg1_done /* synthesis syn_maxfan = 30 **/,
107  output rdlvl_stg1_rnk_done,
108  output reg rdlvl_stg1_err,
109  output mpr_rdlvl_err,
110  output rdlvl_err,
111  output reg rdlvl_prech_req,
112  output reg rdlvl_last_byte_done,
113  output reg rdlvl_assrt_common,
114  input prech_done,
115  input phy_if_empty,
116  input [4:0] idelaye2_init_val,
117  // Captured data in fabric clock domain
118  input [2*nCK_PER_CLK*DQ_WIDTH-1:0] rd_data,
119  // Decrement initial Phaser_IN Fine tap delay
120  input dqs_po_dec_done,
121  input [5:0] pi_counter_read_val,
122  // Stage 1 calibration outputs
123  output reg pi_fine_dly_dec_done,
124  output reg pi_en_stg2_f,
125  output reg pi_stg2_f_incdec,
126  output reg pi_stg2_load,
127  output reg [5:0] pi_stg2_reg_l,
128  output [DQS_CNT_WIDTH:0] pi_stg2_rdlvl_cnt,
129  // To DQ IDELAY required to find left edge of
130  // valid window
131  output idelay_ce,
132  output idelay_inc,
133  input idelay_ld,
134  input [DQS_CNT_WIDTH:0] wrcal_cnt,
135  // Only output if Per-bit de-skew enabled
136  output reg [5*RANKS*DQ_WIDTH-1:0] dlyval_dq,
137  // Debug Port
138  output [6*DQS_WIDTH*RANKS-1:0] dbg_cpt_first_edge_cnt,
139  output [6*DQS_WIDTH*RANKS-1:0] dbg_cpt_second_edge_cnt,
140  output [6*DQS_WIDTH*RANKS-1:0] dbg_cpt_tap_cnt,
141  output [5*DQS_WIDTH*RANKS-1:0] dbg_dq_idelay_tap_cnt,
142 
143  input dbg_idel_up_all,
144  input dbg_idel_down_all,
145  input dbg_idel_up_cpt,
146  input dbg_idel_down_cpt,
147  input [DQS_CNT_WIDTH-1:0] dbg_sel_idel_cpt,
148  input dbg_sel_all_idel_cpt,
149  output [255:0] dbg_phy_rdlvl
150  );
151 
152  // minimum time (in IDELAY taps) for which capture data must be stable for
153  // algorithm to consider a valid data eye to be found. The read leveling
154  // logic will ignore any window found smaller than this value. Limitations
155  // on how small this number can be is determined by: (1) the algorithmic
156  // limitation of how many taps wide the data eye can be (3 taps), and (2)
157  // how wide regions of "instability" that occur around the edges of the
158  // read valid window can be (i.e. need to be able to filter out "false"
159  // windows that occur for a short # of taps around the edges of the true
160  // data window, although with multi-sampling during read leveling, this is
161  // not as much a concern) - the larger the value, the more protection
162  // against "false" windows
163  localparam MIN_EYE_SIZE = 16;
164 
165  // Length of calibration sequence (in # of words)
166  localparam CAL_PAT_LEN = 8;
167  // Read data shift register length
168  localparam RD_SHIFT_LEN = CAL_PAT_LEN / (2*nCK_PER_CLK);
169 
170  // # of cycles required to perform read data shift register compare
171  // This is defined as from the cycle the new data is loaded until
172  // signal found_edge_r is valid
173  localparam RD_SHIFT_COMP_DELAY = 5;
174 
175  // worst-case # of cycles to wait to ensure that both the SR and
176  // PREV_SR shift registers have valid data, and that the comparison
177  // of the two shift register values is valid. The "+1" at the end of
178  // this equation is a fudge factor, I freely admit that
179  localparam SR_VALID_DELAY = (2 * RD_SHIFT_LEN) + RD_SHIFT_COMP_DELAY + 1;
180 
181  // # of clock cycles to wait after changing tap value or read data MUX
182  // to allow: (1) tap chain to settle, (2) for delayed input to propagate
183  // thru ISERDES, (3) for the read data comparison logic to have time to
184  // output the comparison of two consecutive samples of the settled read data
185  // The minimum delay is 16 cycles, which should be good enough to handle all
186  // three of the above conditions for the simulation-only case with a short
187  // training pattern. For H/W (or for simulation with longer training
188  // pattern), it will take longer to store and compare two consecutive
189  // samples, and the value of this parameter will reflect that
190  localparam PIPE_WAIT_CNT = (SR_VALID_DELAY < 8) ? 16 : (SR_VALID_DELAY + 8);
191 
192  // # of read data samples to examine when detecting whether an edge has
193  // occured during stage 1 calibration. Width of local param must be
194  // changed as appropriate. Note that there are two counters used, each
195  // counter can be changed independently of the other - they are used in
196  // cascade to create a larger counter
197  localparam [11:0] DETECT_EDGE_SAMPLE_CNT0 = 12'h001; //12'hFFF;
198  localparam [11:0] DETECT_EDGE_SAMPLE_CNT1 = 12'h001; // 12'h1FF Must be > 0
199 
200  localparam [5:0] CAL1_IDLE = 6'h00;
201  localparam [5:0] CAL1_NEW_DQS_WAIT = 6'h01;
202  localparam [5:0] CAL1_STORE_FIRST_WAIT = 6'h02;
203  localparam [5:0] CAL1_PAT_DETECT = 6'h03;
204  localparam [5:0] CAL1_DQ_IDEL_TAP_INC = 6'h04;
205  localparam [5:0] CAL1_DQ_IDEL_TAP_INC_WAIT = 6'h05;
206  localparam [5:0] CAL1_DQ_IDEL_TAP_DEC = 6'h06;
207  localparam [5:0] CAL1_DQ_IDEL_TAP_DEC_WAIT = 6'h07;
208  localparam [5:0] CAL1_DETECT_EDGE = 6'h08;
209  localparam [5:0] CAL1_IDEL_INC_CPT = 6'h09;
210  localparam [5:0] CAL1_IDEL_INC_CPT_WAIT = 6'h0A;
211  localparam [5:0] CAL1_CALC_IDEL = 6'h0B;
212  localparam [5:0] CAL1_IDEL_DEC_CPT = 6'h0C;
213  localparam [5:0] CAL1_IDEL_DEC_CPT_WAIT = 6'h0D;
214  localparam [5:0] CAL1_NEXT_DQS = 6'h0E;
215  localparam [5:0] CAL1_DONE = 6'h0F;
216  localparam [5:0] CAL1_PB_STORE_FIRST_WAIT = 6'h10;
217  localparam [5:0] CAL1_PB_DETECT_EDGE = 6'h11;
218  localparam [5:0] CAL1_PB_INC_CPT = 6'h12;
219  localparam [5:0] CAL1_PB_INC_CPT_WAIT = 6'h13;
220  localparam [5:0] CAL1_PB_DEC_CPT_LEFT = 6'h14;
221  localparam [5:0] CAL1_PB_DEC_CPT_LEFT_WAIT = 6'h15;
222  localparam [5:0] CAL1_PB_DETECT_EDGE_DQ = 6'h16;
223  localparam [5:0] CAL1_PB_INC_DQ = 6'h17;
224  localparam [5:0] CAL1_PB_INC_DQ_WAIT = 6'h18;
225  localparam [5:0] CAL1_PB_DEC_CPT = 6'h19;
226  localparam [5:0] CAL1_PB_DEC_CPT_WAIT = 6'h1A;
227  localparam [5:0] CAL1_REGL_LOAD = 6'h1B;
228  localparam [5:0] CAL1_RDLVL_ERR = 6'h1C;
229  localparam [5:0] CAL1_MPR_NEW_DQS_WAIT = 6'h1D;
230  localparam [5:0] CAL1_VALID_WAIT = 6'h1E;
231  localparam [5:0] CAL1_MPR_PAT_DETECT = 6'h1F;
232  localparam [5:0] CAL1_NEW_DQS_PREWAIT = 6'h20;
233 
234  integer a;
235  integer b;
236  integer d;
237  integer e;
238  integer f;
239  integer h;
240  integer g;
241  integer i;
242  integer j;
243  integer k;
244  integer l;
245  integer m;
246  integer n;
247  integer r;
248  integer p;
249  integer q;
250  integer s;
251  integer t;
252  integer u;
253  integer w;
254  integer ce_i;
255  integer ce_rnk_i;
256  integer aa;
257  integer bb;
258  integer cc;
259  integer dd;
260  genvar x;
261  genvar z;
262 
263  reg [DQS_CNT_WIDTH:0] cal1_cnt_cpt_r;
264  wire [DQS_CNT_WIDTH+2:0]cal1_cnt_cpt_timing;
265  reg [DQS_CNT_WIDTH:0] cal1_cnt_cpt_timing_r;
266  reg cal1_dq_idel_ce;
267  reg cal1_dq_idel_inc;
268  reg cal1_dlyce_cpt_r;
269  reg cal1_dlyinc_cpt_r;
270  reg cal1_dlyce_dq_r;
271  reg cal1_dlyinc_dq_r;
272  reg cal1_wait_cnt_en_r;
273  reg [4:0] cal1_wait_cnt_r;
274  reg cal1_wait_r;
275  reg [DQ_WIDTH-1:0] dlyce_dq_r;
276  reg dlyinc_dq_r;
277  reg [4:0] dlyval_dq_reg_r [0:RANKS-1][0:DQ_WIDTH-1];
278  reg cal1_prech_req_r;
279  reg [5:0] cal1_state_r;
280  reg [5:0] cal1_state_r1;
281  reg [5:0] cnt_idel_dec_cpt_r;
282  reg [3:0] cnt_shift_r;
283  reg detect_edge_done_r;
284  reg [5:0] right_edge_taps_r;
285  reg [5:0] first_edge_taps_r;
286  reg found_edge_r;
287  reg found_first_edge_r;
288  reg found_second_edge_r;
289  reg found_stable_eye_r;
290  reg found_stable_eye_last_r;
291  reg found_edge_all_r;
292  reg [5:0] tap_cnt_cpt_r;
293  reg tap_limit_cpt_r;
294  reg [4:0] idel_tap_cnt_dq_pb_r;
295  reg idel_tap_limit_dq_pb_r;
296  reg [DRAM_WIDTH-1:0] mux_rd_fall0_r;
297  reg [DRAM_WIDTH-1:0] mux_rd_fall1_r;
298  reg [DRAM_WIDTH-1:0] mux_rd_rise0_r;
299  reg [DRAM_WIDTH-1:0] mux_rd_rise1_r;
300  reg [DRAM_WIDTH-1:0] mux_rd_fall2_r;
301  reg [DRAM_WIDTH-1:0] mux_rd_fall3_r;
302  reg [DRAM_WIDTH-1:0] mux_rd_rise2_r;
303  reg [DRAM_WIDTH-1:0] mux_rd_rise3_r;
304  reg mux_rd_valid_r;
305  reg new_cnt_cpt_r;
306  reg [RD_SHIFT_LEN-1:0] old_sr_fall0_r [DRAM_WIDTH-1:0];
307  reg [RD_SHIFT_LEN-1:0] old_sr_fall1_r [DRAM_WIDTH-1:0];
308  reg [RD_SHIFT_LEN-1:0] old_sr_rise0_r [DRAM_WIDTH-1:0];
309  reg [RD_SHIFT_LEN-1:0] old_sr_rise1_r [DRAM_WIDTH-1:0];
310  reg [RD_SHIFT_LEN-1:0] old_sr_fall2_r [DRAM_WIDTH-1:0];
311  reg [RD_SHIFT_LEN-1:0] old_sr_fall3_r [DRAM_WIDTH-1:0];
312  reg [RD_SHIFT_LEN-1:0] old_sr_rise2_r [DRAM_WIDTH-1:0];
313  reg [RD_SHIFT_LEN-1:0] old_sr_rise3_r [DRAM_WIDTH-1:0];
314  reg [DRAM_WIDTH-1:0] old_sr_match_fall0_r;
315  reg [DRAM_WIDTH-1:0] old_sr_match_fall1_r;
316  reg [DRAM_WIDTH-1:0] old_sr_match_rise0_r;
317  reg [DRAM_WIDTH-1:0] old_sr_match_rise1_r;
318  reg [DRAM_WIDTH-1:0] old_sr_match_fall2_r;
319  reg [DRAM_WIDTH-1:0] old_sr_match_fall3_r;
320  reg [DRAM_WIDTH-1:0] old_sr_match_rise2_r;
321  reg [DRAM_WIDTH-1:0] old_sr_match_rise3_r;
322  reg [4:0] pb_cnt_eye_size_r [DRAM_WIDTH-1:0];
323  reg [DRAM_WIDTH-1:0] pb_detect_edge_done_r;
324  reg [DRAM_WIDTH-1:0] pb_found_edge_last_r;
325  reg [DRAM_WIDTH-1:0] pb_found_edge_r;
326  reg [DRAM_WIDTH-1:0] pb_found_first_edge_r;
327  reg [DRAM_WIDTH-1:0] pb_found_stable_eye_r;
328  reg [DRAM_WIDTH-1:0] pb_last_tap_jitter_r;
329  reg pi_en_stg2_f_timing;
330  reg pi_stg2_f_incdec_timing;
331  reg pi_stg2_load_timing;
332  reg [5:0] pi_stg2_reg_l_timing;
333  reg [DRAM_WIDTH-1:0] prev_sr_diff_r;
334  reg [RD_SHIFT_LEN-1:0] prev_sr_fall0_r [DRAM_WIDTH-1:0];
335  reg [RD_SHIFT_LEN-1:0] prev_sr_fall1_r [DRAM_WIDTH-1:0];
336  reg [RD_SHIFT_LEN-1:0] prev_sr_rise0_r [DRAM_WIDTH-1:0];
337  reg [RD_SHIFT_LEN-1:0] prev_sr_rise1_r [DRAM_WIDTH-1:0];
338  reg [RD_SHIFT_LEN-1:0] prev_sr_fall2_r [DRAM_WIDTH-1:0];
339  reg [RD_SHIFT_LEN-1:0] prev_sr_fall3_r [DRAM_WIDTH-1:0];
340  reg [RD_SHIFT_LEN-1:0] prev_sr_rise2_r [DRAM_WIDTH-1:0];
341  reg [RD_SHIFT_LEN-1:0] prev_sr_rise3_r [DRAM_WIDTH-1:0];
342  reg [DRAM_WIDTH-1:0] prev_sr_match_cyc2_r;
343  reg [DRAM_WIDTH-1:0] prev_sr_match_fall0_r;
344  reg [DRAM_WIDTH-1:0] prev_sr_match_fall1_r;
345  reg [DRAM_WIDTH-1:0] prev_sr_match_rise0_r;
346  reg [DRAM_WIDTH-1:0] prev_sr_match_rise1_r;
347  reg [DRAM_WIDTH-1:0] prev_sr_match_fall2_r;
348  reg [DRAM_WIDTH-1:0] prev_sr_match_fall3_r;
349  reg [DRAM_WIDTH-1:0] prev_sr_match_rise2_r;
350  reg [DRAM_WIDTH-1:0] prev_sr_match_rise3_r;
351  wire [DQ_WIDTH-1:0] rd_data_rise0;
352  wire [DQ_WIDTH-1:0] rd_data_fall0;
353  wire [DQ_WIDTH-1:0] rd_data_rise1;
354  wire [DQ_WIDTH-1:0] rd_data_fall1;
355  wire [DQ_WIDTH-1:0] rd_data_rise2;
356  wire [DQ_WIDTH-1:0] rd_data_fall2;
357  wire [DQ_WIDTH-1:0] rd_data_rise3;
358  wire [DQ_WIDTH-1:0] rd_data_fall3;
359  reg samp_cnt_done_r;
360  reg samp_edge_cnt0_en_r;
361  reg [11:0] samp_edge_cnt0_r;
362  reg samp_edge_cnt1_en_r;
363  reg [11:0] samp_edge_cnt1_r;
364  reg [DQS_CNT_WIDTH:0] rd_mux_sel_r;
365  reg [5:0] second_edge_taps_r;
366  reg [RD_SHIFT_LEN-1:0] sr_fall0_r [DRAM_WIDTH-1:0];
367  reg [RD_SHIFT_LEN-1:0] sr_fall1_r [DRAM_WIDTH-1:0];
368  reg [RD_SHIFT_LEN-1:0] sr_rise0_r [DRAM_WIDTH-1:0];
369  reg [RD_SHIFT_LEN-1:0] sr_rise1_r [DRAM_WIDTH-1:0];
370  reg [RD_SHIFT_LEN-1:0] sr_fall2_r [DRAM_WIDTH-1:0];
371  reg [RD_SHIFT_LEN-1:0] sr_fall3_r [DRAM_WIDTH-1:0];
372  reg [RD_SHIFT_LEN-1:0] sr_rise2_r [DRAM_WIDTH-1:0];
373  reg [RD_SHIFT_LEN-1:0] sr_rise3_r [DRAM_WIDTH-1:0];
374  reg store_sr_r;
375  reg store_sr_req_pulsed_r;
376  reg store_sr_req_r;
377  reg sr_valid_r;
378  reg sr_valid_r1;
379  reg sr_valid_r2;
380  reg [DRAM_WIDTH-1:0] old_sr_diff_r;
381  reg [DRAM_WIDTH-1:0] old_sr_match_cyc2_r;
382  reg pat0_data_match_r;
383  reg pat1_data_match_r;
384  wire pat_data_match_r;
385  wire [RD_SHIFT_LEN-1:0] pat0_fall0 [3:0];
386  wire [RD_SHIFT_LEN-1:0] pat0_fall1 [3:0];
387  wire [RD_SHIFT_LEN-1:0] pat0_fall2 [3:0];
388  wire [RD_SHIFT_LEN-1:0] pat0_fall3 [3:0];
389  wire [RD_SHIFT_LEN-1:0] pat1_fall0 [3:0];
390  wire [RD_SHIFT_LEN-1:0] pat1_fall1 [3:0];
391  wire [RD_SHIFT_LEN-1:0] pat1_fall2 [3:0];
392  wire [RD_SHIFT_LEN-1:0] pat1_fall3 [3:0];
393  reg [DRAM_WIDTH-1:0] pat0_match_fall0_r;
394  reg pat0_match_fall0_and_r;
395  reg [DRAM_WIDTH-1:0] pat0_match_fall1_r;
396  reg pat0_match_fall1_and_r;
397  reg [DRAM_WIDTH-1:0] pat0_match_fall2_r;
398  reg pat0_match_fall2_and_r;
399  reg [DRAM_WIDTH-1:0] pat0_match_fall3_r;
400  reg pat0_match_fall3_and_r;
401  reg [DRAM_WIDTH-1:0] pat0_match_rise0_r;
402  reg pat0_match_rise0_and_r;
403  reg [DRAM_WIDTH-1:0] pat0_match_rise1_r;
404  reg pat0_match_rise1_and_r;
405  reg [DRAM_WIDTH-1:0] pat0_match_rise2_r;
406  reg pat0_match_rise2_and_r;
407  reg [DRAM_WIDTH-1:0] pat0_match_rise3_r;
408  reg pat0_match_rise3_and_r;
409  reg [DRAM_WIDTH-1:0] pat1_match_fall0_r;
410  reg pat1_match_fall0_and_r;
411  reg [DRAM_WIDTH-1:0] pat1_match_fall1_r;
412  reg pat1_match_fall1_and_r;
413  reg [DRAM_WIDTH-1:0] pat1_match_fall2_r;
414  reg pat1_match_fall2_and_r;
415  reg [DRAM_WIDTH-1:0] pat1_match_fall3_r;
416  reg pat1_match_fall3_and_r;
417  reg [DRAM_WIDTH-1:0] pat1_match_rise0_r;
418  reg pat1_match_rise0_and_r;
419  reg [DRAM_WIDTH-1:0] pat1_match_rise1_r;
420  reg pat1_match_rise1_and_r;
421  reg [DRAM_WIDTH-1:0] pat1_match_rise2_r;
422  reg pat1_match_rise2_and_r;
423  reg [DRAM_WIDTH-1:0] pat1_match_rise3_r;
424  reg pat1_match_rise3_and_r;
425  reg [4:0] idelay_tap_cnt_r [0:RANKS-1][0:DQS_WIDTH-1];
426  reg [5*DQS_WIDTH*RANKS-1:0] idelay_tap_cnt_w;
427  reg [4:0] idelay_tap_cnt_slice_r;
428  reg idelay_tap_limit_r;
429 
430  wire [RD_SHIFT_LEN-1:0] pat0_rise0 [3:0];
431  wire [RD_SHIFT_LEN-1:0] pat0_rise1 [3:0];
432  wire [RD_SHIFT_LEN-1:0] pat0_rise2 [3:0];
433  wire [RD_SHIFT_LEN-1:0] pat0_rise3 [3:0];
434  wire [RD_SHIFT_LEN-1:0] pat1_rise0 [3:0];
435  wire [RD_SHIFT_LEN-1:0] pat1_rise1 [3:0];
436  wire [RD_SHIFT_LEN-1:0] pat1_rise2 [3:0];
437  wire [RD_SHIFT_LEN-1:0] pat1_rise3 [3:0];
438 
439  wire [RD_SHIFT_LEN-1:0] idel_pat0_rise0 [3:0];
440  wire [RD_SHIFT_LEN-1:0] idel_pat0_fall0 [3:0];
441  wire [RD_SHIFT_LEN-1:0] idel_pat0_rise1 [3:0];
442  wire [RD_SHIFT_LEN-1:0] idel_pat0_fall1 [3:0];
443  wire [RD_SHIFT_LEN-1:0] idel_pat0_rise2 [3:0];
444  wire [RD_SHIFT_LEN-1:0] idel_pat0_fall2 [3:0];
445  wire [RD_SHIFT_LEN-1:0] idel_pat0_rise3 [3:0];
446  wire [RD_SHIFT_LEN-1:0] idel_pat0_fall3 [3:0];
447  wire [RD_SHIFT_LEN-1:0] idel_pat1_rise0 [3:0];
448  wire [RD_SHIFT_LEN-1:0] idel_pat1_fall0 [3:0];
449  wire [RD_SHIFT_LEN-1:0] idel_pat1_rise1 [3:0];
450  wire [RD_SHIFT_LEN-1:0] idel_pat1_fall1 [3:0];
451  wire [RD_SHIFT_LEN-1:0] idel_pat1_rise2 [3:0];
452  wire [RD_SHIFT_LEN-1:0] idel_pat1_fall2 [3:0];
453  wire [RD_SHIFT_LEN-1:0] idel_pat1_rise3 [3:0];
454  wire [RD_SHIFT_LEN-1:0] idel_pat1_fall3 [3:0];
455 
456  reg [DRAM_WIDTH-1:0] idel_pat0_match_rise0_r;
457  reg [DRAM_WIDTH-1:0] idel_pat0_match_fall0_r;
458  reg [DRAM_WIDTH-1:0] idel_pat0_match_rise1_r;
459  reg [DRAM_WIDTH-1:0] idel_pat0_match_fall1_r;
460  reg [DRAM_WIDTH-1:0] idel_pat0_match_rise2_r;
461  reg [DRAM_WIDTH-1:0] idel_pat0_match_fall2_r;
462  reg [DRAM_WIDTH-1:0] idel_pat0_match_rise3_r;
463  reg [DRAM_WIDTH-1:0] idel_pat0_match_fall3_r;
464 
465  reg [DRAM_WIDTH-1:0] idel_pat1_match_rise0_r;
466  reg [DRAM_WIDTH-1:0] idel_pat1_match_fall0_r;
467  reg [DRAM_WIDTH-1:0] idel_pat1_match_rise1_r;
468  reg [DRAM_WIDTH-1:0] idel_pat1_match_fall1_r;
469  reg [DRAM_WIDTH-1:0] idel_pat1_match_rise2_r;
470  reg [DRAM_WIDTH-1:0] idel_pat1_match_fall2_r;
471  reg [DRAM_WIDTH-1:0] idel_pat1_match_rise3_r;
472  reg [DRAM_WIDTH-1:0] idel_pat1_match_fall3_r;
473 
474  reg idel_pat0_match_rise0_and_r;
475  reg idel_pat0_match_fall0_and_r;
476  reg idel_pat0_match_rise1_and_r;
477  reg idel_pat0_match_fall1_and_r;
478  reg idel_pat0_match_rise2_and_r;
479  reg idel_pat0_match_fall2_and_r;
480  reg idel_pat0_match_rise3_and_r;
481  reg idel_pat0_match_fall3_and_r;
482 
483  reg idel_pat1_match_rise0_and_r;
484  reg idel_pat1_match_fall0_and_r;
485  reg idel_pat1_match_rise1_and_r;
486  reg idel_pat1_match_fall1_and_r;
487  reg idel_pat1_match_rise2_and_r;
488  reg idel_pat1_match_fall2_and_r;
489  reg idel_pat1_match_rise3_and_r;
490  reg idel_pat1_match_fall3_and_r;
491 
492  reg idel_pat0_data_match_r;
493  reg idel_pat1_data_match_r;
494 
495  reg idel_pat_data_match;
496  reg idel_pat_data_match_r;
497 
498  reg [4:0] idel_dec_cnt;
499 
500  reg [5:0] rdlvl_dqs_tap_cnt_r [0:RANKS-1][0:DQS_WIDTH-1];
501  reg [1:0] rnk_cnt_r;
502  reg rdlvl_rank_done_r;
503 
504  reg [3:0] done_cnt;
505  reg [1:0] regl_rank_cnt;
506  reg [DQS_CNT_WIDTH:0] regl_dqs_cnt;
507  reg [DQS_CNT_WIDTH:0] regl_dqs_cnt_r;
508  wire [DQS_CNT_WIDTH+2:0]regl_dqs_cnt_timing;
509  reg regl_rank_done_r;
510  reg rdlvl_stg1_start_r;
511 
512  reg dqs_po_dec_done_r1;
513  reg dqs_po_dec_done_r2;
514  reg fine_dly_dec_done_r1;
515  reg fine_dly_dec_done_r2;
516  reg [3:0] wait_cnt_r;
517  reg [5:0] pi_rdval_cnt;
518  reg pi_cnt_dec;
519 
520  reg mpr_valid_r;
521  reg mpr_valid_r1;
522  reg mpr_valid_r2;
523  reg mpr_rd_rise0_prev_r;
524  reg mpr_rd_fall0_prev_r;
525  reg mpr_rd_rise1_prev_r;
526  reg mpr_rd_fall1_prev_r;
527  reg mpr_rd_rise2_prev_r;
528  reg mpr_rd_fall2_prev_r;
529  reg mpr_rd_rise3_prev_r;
530  reg mpr_rd_fall3_prev_r;
531  reg mpr_rdlvl_done_r;
532  reg mpr_rdlvl_done_r1;
533  reg mpr_rdlvl_done_r2;
534  reg mpr_rdlvl_start_r;
535  reg mpr_rank_done_r;
536  reg [2:0] stable_idel_cnt;
537  reg inhibit_edge_detect_r;
538  reg idel_pat_detect_valid_r;
539  reg idel_mpr_pat_detect_r;
540  reg mpr_pat_detect_r;
541  reg mpr_dec_cpt_r;
542 
543 
544  // Debug
545  reg [6*DQS_WIDTH*RANKS-1:0] dbg_cpt_first_edge_taps;
546  reg [6*DQS_WIDTH*RANKS-1:0] dbg_cpt_second_edge_taps;
547  reg [6*DQS_WIDTH*RANKS-1:0] dbg_cpt_tap_cnt_w;
548 
549  //***************************************************************************
550  // Debug
551  //***************************************************************************
552 
553  always @(*) begin
554  for (d = 0; d < RANKS; d = d + 1) begin
555  for (e = 0; e < DQS_WIDTH; e = e + 1) begin
556  idelay_tap_cnt_w[(5*e+5*DQS_WIDTH*d)+:5] <= #TCQ idelay_tap_cnt_r[d][e];
557  dbg_cpt_tap_cnt_w[(6*e+6*DQS_WIDTH*d)+:6] <= #TCQ rdlvl_dqs_tap_cnt_r[d][e];
558  end
559  end
560  end
561 
562  assign mpr_rdlvl_err = rdlvl_stg1_err & (!mpr_rdlvl_done);
563  assign rdlvl_err = rdlvl_stg1_err & (mpr_rdlvl_done);
564 
565 
566  assign dbg_phy_rdlvl[0] = rdlvl_stg1_start;
567  assign dbg_phy_rdlvl[1] = pat_data_match_r;
568  assign dbg_phy_rdlvl[2] = mux_rd_valid_r;
569  assign dbg_phy_rdlvl[3] = idelay_tap_limit_r;
570  assign dbg_phy_rdlvl[8:4] = 'b0;
571  assign dbg_phy_rdlvl[14:9] = cal1_state_r[5:0];
572  assign dbg_phy_rdlvl[20:15] = cnt_idel_dec_cpt_r;
573  assign dbg_phy_rdlvl[21] = found_first_edge_r;
574  assign dbg_phy_rdlvl[22] = found_second_edge_r;
575  assign dbg_phy_rdlvl[23] = found_edge_r;
576  assign dbg_phy_rdlvl[24] = store_sr_r;
577  // [40:25] previously used for sr, old_sr shift registers. If connecting
578  // these signals again, don't forget to parameterize based on RD_SHIFT_LEN
579  assign dbg_phy_rdlvl[40:25] = 'b0;
580  assign dbg_phy_rdlvl[41] = sr_valid_r;
581  assign dbg_phy_rdlvl[42] = found_stable_eye_r;
582  assign dbg_phy_rdlvl[48:43] = tap_cnt_cpt_r;
583  assign dbg_phy_rdlvl[54:49] = first_edge_taps_r;
584  assign dbg_phy_rdlvl[60:55] = second_edge_taps_r;
585  assign dbg_phy_rdlvl[64:61] = cal1_cnt_cpt_timing_r;
586  assign dbg_phy_rdlvl[65] = cal1_dlyce_cpt_r;
587  assign dbg_phy_rdlvl[66] = cal1_dlyinc_cpt_r;
588  assign dbg_phy_rdlvl[67] = found_edge_r;
589  assign dbg_phy_rdlvl[68] = found_first_edge_r;
590  assign dbg_phy_rdlvl[73:69] = 'b0;
591  assign dbg_phy_rdlvl[74] = idel_pat_data_match;
592  assign dbg_phy_rdlvl[75] = idel_pat0_data_match_r;
593  assign dbg_phy_rdlvl[76] = idel_pat1_data_match_r;
594  assign dbg_phy_rdlvl[77] = pat0_data_match_r;
595  assign dbg_phy_rdlvl[78] = pat1_data_match_r;
596  assign dbg_phy_rdlvl[79+:5*DQS_WIDTH*RANKS] = idelay_tap_cnt_w;
597  assign dbg_phy_rdlvl[170+:8] = mux_rd_rise0_r;
598  assign dbg_phy_rdlvl[178+:8] = mux_rd_fall0_r;
599  assign dbg_phy_rdlvl[186+:8] = mux_rd_rise1_r;
600  assign dbg_phy_rdlvl[194+:8] = mux_rd_fall1_r;
601  assign dbg_phy_rdlvl[202+:8] = mux_rd_rise2_r;
602  assign dbg_phy_rdlvl[210+:8] = mux_rd_fall2_r;
603  assign dbg_phy_rdlvl[218+:8] = mux_rd_rise3_r;
604  assign dbg_phy_rdlvl[226+:8] = mux_rd_fall3_r;
605 
606  //***************************************************************************
607  // Debug output
608  //***************************************************************************
609 
610  // CPT taps
611  assign dbg_cpt_first_edge_cnt = dbg_cpt_first_edge_taps;
612  assign dbg_cpt_second_edge_cnt = dbg_cpt_second_edge_taps;
613  assign dbg_cpt_tap_cnt = dbg_cpt_tap_cnt_w;
614  assign dbg_dq_idelay_tap_cnt = idelay_tap_cnt_w;
615 
616  // Record first and second edges found during CPT calibration
617 
618  generate
619  always @(posedge clk)
620  if (rst) begin
621  dbg_cpt_first_edge_taps <= #TCQ 'b0;
622  dbg_cpt_second_edge_taps <= #TCQ 'b0;
623  end else if ((SIM_CAL_OPTION == "FAST_CAL") & (cal1_state_r1 == CAL1_CALC_IDEL)) begin
624  for (ce_rnk_i = 0; ce_rnk_i < RANKS; ce_rnk_i = ce_rnk_i + 1) begin: gen_dbg_cpt_rnk
625  for (ce_i = 0; ce_i < DQS_WIDTH; ce_i = ce_i + 1) begin: gen_dbg_cpt_edge
626  if (found_first_edge_r)
627  dbg_cpt_first_edge_taps[((6*ce_i)+(ce_rnk_i*DQS_WIDTH*6))+:6]
628  <= #TCQ first_edge_taps_r;
629  if (found_second_edge_r)
630  dbg_cpt_second_edge_taps[((6*ce_i)+(ce_rnk_i*DQS_WIDTH*6))+:6]
631  <= #TCQ second_edge_taps_r;
632  end
633  end
634  end else if (cal1_state_r == CAL1_CALC_IDEL) begin
635  // Record tap counts of first and second edge edges during
636  // CPT calibration for each DQS group. If neither edge has
637  // been found, then those taps will remain 0
638  if (found_first_edge_r)
639  dbg_cpt_first_edge_taps[(((cal1_cnt_cpt_timing <<2) + (cal1_cnt_cpt_timing <<1))
640  +(rnk_cnt_r*DQS_WIDTH*6))+:6]
641  <= #TCQ first_edge_taps_r;
642  if (found_second_edge_r)
643  dbg_cpt_second_edge_taps[(((cal1_cnt_cpt_timing <<2) + (cal1_cnt_cpt_timing <<1))
644  +(rnk_cnt_r*DQS_WIDTH*6))+:6]
645  <= #TCQ second_edge_taps_r;
646  end
647  endgenerate
648 
649  assign rdlvl_stg1_rnk_done = rdlvl_rank_done_r;// || regl_rank_done_r;
650  assign mpr_rnk_done = mpr_rank_done_r;
651  assign mpr_rdlvl_done = ((DRAM_TYPE == "DDR3") && (OCAL_EN == "ON")) ? //&& (SIM_CAL_OPTION == "NONE")
652  mpr_rdlvl_done_r : 1'b1;
653 
654  //**************************************************************************
655  // DQS count to hard PHY during write calibration using Phaser_OUT Stage2
656  // coarse delay
657  //**************************************************************************
658  assign pi_stg2_rdlvl_cnt = (cal1_state_r == CAL1_REGL_LOAD) ? regl_dqs_cnt_r : cal1_cnt_cpt_r;
659 
660  assign idelay_ce = cal1_dq_idel_ce;
661  assign idelay_inc = cal1_dq_idel_inc;
662 
663  //***************************************************************************
664  // Assert calib_in_common in FAST_CAL mode for IDELAY tap increments to all
665  // DQs simultaneously
666  //***************************************************************************
667 
668  always @(posedge clk) begin
669  if (rst)
670  rdlvl_assrt_common <= #TCQ 1'b0;
671  else if ((SIM_CAL_OPTION == "FAST_CAL") & rdlvl_stg1_start &
672  !rdlvl_stg1_start_r)
673  rdlvl_assrt_common <= #TCQ 1'b1;
674  else if (!idel_pat_data_match_r & idel_pat_data_match)
675  rdlvl_assrt_common <= #TCQ 1'b0;
676  end
677 
678  //***************************************************************************
679  // Data mux to route appropriate bit to calibration logic - i.e. calibration
680  // is done sequentially, one bit (or DQS group) at a time
681  //***************************************************************************
682 
683  generate
684  if (nCK_PER_CLK == 4) begin: rd_data_div4_logic_clk
685  assign rd_data_rise0 = rd_data[DQ_WIDTH-1:0];
686  assign rd_data_fall0 = rd_data[2*DQ_WIDTH-1:DQ_WIDTH];
687  assign rd_data_rise1 = rd_data[3*DQ_WIDTH-1:2*DQ_WIDTH];
688  assign rd_data_fall1 = rd_data[4*DQ_WIDTH-1:3*DQ_WIDTH];
689  assign rd_data_rise2 = rd_data[5*DQ_WIDTH-1:4*DQ_WIDTH];
690  assign rd_data_fall2 = rd_data[6*DQ_WIDTH-1:5*DQ_WIDTH];
691  assign rd_data_rise3 = rd_data[7*DQ_WIDTH-1:6*DQ_WIDTH];
692  assign rd_data_fall3 = rd_data[8*DQ_WIDTH-1:7*DQ_WIDTH];
693  end else begin: rd_data_div2_logic_clk
694  assign rd_data_rise0 = rd_data[DQ_WIDTH-1:0];
695  assign rd_data_fall0 = rd_data[2*DQ_WIDTH-1:DQ_WIDTH];
696  assign rd_data_rise1 = rd_data[3*DQ_WIDTH-1:2*DQ_WIDTH];
697  assign rd_data_fall1 = rd_data[4*DQ_WIDTH-1:3*DQ_WIDTH];
698  end
699  endgenerate
700 
701  always @(posedge clk) begin
702  rd_mux_sel_r <= #TCQ cal1_cnt_cpt_r;
703  end
704 
705  // Register outputs for improved timing.
706  // NOTE: Will need to change when per-bit DQ deskew is supported.
707  // Currenly all bits in DQS group are checked in aggregate
708  generate
709  genvar mux_i;
710  for (mux_i = 0; mux_i < DRAM_WIDTH; mux_i = mux_i + 1) begin: gen_mux_rd
711  always @(posedge clk) begin
712  mux_rd_rise0_r[mux_i] <= #TCQ rd_data_rise0[DRAM_WIDTH*rd_mux_sel_r +
713  mux_i];
714  mux_rd_fall0_r[mux_i] <= #TCQ rd_data_fall0[DRAM_WIDTH*rd_mux_sel_r +
715  mux_i];
716  mux_rd_rise1_r[mux_i] <= #TCQ rd_data_rise1[DRAM_WIDTH*rd_mux_sel_r +
717  mux_i];
718  mux_rd_fall1_r[mux_i] <= #TCQ rd_data_fall1[DRAM_WIDTH*rd_mux_sel_r +
719  mux_i];
720  mux_rd_rise2_r[mux_i] <= #TCQ rd_data_rise2[DRAM_WIDTH*rd_mux_sel_r +
721  mux_i];
722  mux_rd_fall2_r[mux_i] <= #TCQ rd_data_fall2[DRAM_WIDTH*rd_mux_sel_r +
723  mux_i];
724  mux_rd_rise3_r[mux_i] <= #TCQ rd_data_rise3[DRAM_WIDTH*rd_mux_sel_r +
725  mux_i];
726  mux_rd_fall3_r[mux_i] <= #TCQ rd_data_fall3[DRAM_WIDTH*rd_mux_sel_r +
727  mux_i];
728  end
729  end
730  endgenerate
731 
732  //***************************************************************************
733  // MPR Read Leveling
734  //***************************************************************************
735 
736  // storing the previous read data for checking later. Only bit 0 is used
737  // since MPR contents (01010101) are available generally on DQ[0] per
738  // JEDEC spec.
739  always @(posedge clk)begin
740  if ((cal1_state_r == CAL1_MPR_NEW_DQS_WAIT) ||
741  ((cal1_state_r == CAL1_MPR_PAT_DETECT) && (idel_pat_detect_valid_r)))begin
742  mpr_rd_rise0_prev_r <= #TCQ mux_rd_rise0_r[0];
743  mpr_rd_fall0_prev_r <= #TCQ mux_rd_fall0_r[0];
744  mpr_rd_rise1_prev_r <= #TCQ mux_rd_rise1_r[0];
745  mpr_rd_fall1_prev_r <= #TCQ mux_rd_fall1_r[0];
746  mpr_rd_rise2_prev_r <= #TCQ mux_rd_rise2_r[0];
747  mpr_rd_fall2_prev_r <= #TCQ mux_rd_fall2_r[0];
748  mpr_rd_rise3_prev_r <= #TCQ mux_rd_rise3_r[0];
749  mpr_rd_fall3_prev_r <= #TCQ mux_rd_fall3_r[0];
750  end
751  end
752 
753  generate
754  if (nCK_PER_CLK == 4) begin: mpr_4to1
755  // changed stable count of 2 IDELAY taps at 78 ps resolution
756  always @(posedge clk) begin
757  if (rst | (cal1_state_r == CAL1_NEW_DQS_PREWAIT) |
758  //(cal1_state_r == CAL1_DETECT_EDGE) |
759  (mpr_rd_rise0_prev_r != mux_rd_rise0_r[0]) |
760  (mpr_rd_fall0_prev_r != mux_rd_fall0_r[0]) |
761  (mpr_rd_rise1_prev_r != mux_rd_rise1_r[0]) |
762  (mpr_rd_fall1_prev_r != mux_rd_fall1_r[0]) |
763  (mpr_rd_rise2_prev_r != mux_rd_rise2_r[0]) |
764  (mpr_rd_fall2_prev_r != mux_rd_fall2_r[0]) |
765  (mpr_rd_rise3_prev_r != mux_rd_rise3_r[0]) |
766  (mpr_rd_fall3_prev_r != mux_rd_fall3_r[0]))
767  stable_idel_cnt <= #TCQ 3'd0;
768  else if ((|idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing]) &
769  ((cal1_state_r == CAL1_MPR_PAT_DETECT) &
770  (idel_pat_detect_valid_r))) begin
771  if ((mpr_rd_rise0_prev_r == mux_rd_rise0_r[0]) &
772  (mpr_rd_fall0_prev_r == mux_rd_fall0_r[0]) &
773  (mpr_rd_rise1_prev_r == mux_rd_rise1_r[0]) &
774  (mpr_rd_fall1_prev_r == mux_rd_fall1_r[0]) &
775  (mpr_rd_rise2_prev_r == mux_rd_rise2_r[0]) &
776  (mpr_rd_fall2_prev_r == mux_rd_fall2_r[0]) &
777  (mpr_rd_rise3_prev_r == mux_rd_rise3_r[0]) &
778  (mpr_rd_fall3_prev_r == mux_rd_fall3_r[0]) &
779  (stable_idel_cnt < 3'd2))
780  stable_idel_cnt <= #TCQ stable_idel_cnt + 1;
781  end
782  end
783 
784  always @(posedge clk) begin
785  if (rst |
786  (mpr_rd_rise0_prev_r & ~mpr_rd_fall0_prev_r &
787  mpr_rd_rise1_prev_r & ~mpr_rd_fall1_prev_r &
788  mpr_rd_rise2_prev_r & ~mpr_rd_fall2_prev_r &
789  mpr_rd_rise3_prev_r & ~mpr_rd_fall3_prev_r))
790  inhibit_edge_detect_r <= 1'b1;
791  // Wait for settling time after idelay tap increment before
792  // de-asserting inhibit_edge_detect_r
793  else if ((cal1_state_r == CAL1_MPR_PAT_DETECT) &
794  (idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing] > 5'd1) &
795  (~mpr_rd_rise0_prev_r & mpr_rd_fall0_prev_r &
796  ~mpr_rd_rise1_prev_r & mpr_rd_fall1_prev_r &
797  ~mpr_rd_rise2_prev_r & mpr_rd_fall2_prev_r &
798  ~mpr_rd_rise3_prev_r & mpr_rd_fall3_prev_r))
799  inhibit_edge_detect_r <= 1'b0;
800  end
801 
802  //checking for transition from 01010101 to 10101010
803  always @(posedge clk)begin
804  if (rst | (cal1_state_r == CAL1_MPR_NEW_DQS_WAIT) |
805  inhibit_edge_detect_r)
806  idel_mpr_pat_detect_r <= #TCQ 1'b0;
807  // 10101010 is not the correct pattern
808  else if ((mpr_rd_rise0_prev_r & ~mpr_rd_fall0_prev_r &
809  mpr_rd_rise1_prev_r & ~mpr_rd_fall1_prev_r &
810  mpr_rd_rise2_prev_r & ~mpr_rd_fall2_prev_r &
811  mpr_rd_rise3_prev_r & ~mpr_rd_fall3_prev_r) ||
812  ((stable_idel_cnt < 3'd2) & (cal1_state_r == CAL1_MPR_PAT_DETECT)
813  && (idel_pat_detect_valid_r)))
814  //|| (idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing] < 5'd2))
815  idel_mpr_pat_detect_r <= #TCQ 1'b0;
816  // 01010101 to 10101010 is the correct transition
817  else if ((~mpr_rd_rise0_prev_r & mpr_rd_fall0_prev_r &
818  ~mpr_rd_rise1_prev_r & mpr_rd_fall1_prev_r &
819  ~mpr_rd_rise2_prev_r & mpr_rd_fall2_prev_r &
820  ~mpr_rd_rise3_prev_r & mpr_rd_fall3_prev_r) &
821  (stable_idel_cnt == 3'd2) &
822  ((mpr_rd_rise0_prev_r != mux_rd_rise0_r[0]) ||
823  (mpr_rd_fall0_prev_r != mux_rd_fall0_r[0]) ||
824  (mpr_rd_rise1_prev_r != mux_rd_rise1_r[0]) ||
825  (mpr_rd_fall1_prev_r != mux_rd_fall1_r[0]) ||
826  (mpr_rd_rise2_prev_r != mux_rd_rise2_r[0]) ||
827  (mpr_rd_fall2_prev_r != mux_rd_fall2_r[0]) ||
828  (mpr_rd_rise3_prev_r != mux_rd_rise3_r[0]) ||
829  (mpr_rd_fall3_prev_r != mux_rd_fall3_r[0])))
830  idel_mpr_pat_detect_r <= #TCQ 1'b1;
831  end
832  end else if (nCK_PER_CLK == 2) begin: mpr_2to1
833  // changed stable count of 2 IDELAY taps at 78 ps resolution
834  always @(posedge clk) begin
835  if (rst | (cal1_state_r == CAL1_MPR_NEW_DQS_WAIT) |
836  (mpr_rd_rise0_prev_r != mux_rd_rise0_r[0]) |
837  (mpr_rd_fall0_prev_r != mux_rd_fall0_r[0]) |
838  (mpr_rd_rise1_prev_r != mux_rd_rise1_r[0]) |
839  (mpr_rd_fall1_prev_r != mux_rd_fall1_r[0]))
840  stable_idel_cnt <= #TCQ 3'd0;
841  else if ((idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing] > 5'd0) &
842  ((cal1_state_r == CAL1_MPR_PAT_DETECT) &
843  (idel_pat_detect_valid_r))) begin
844  if ((mpr_rd_rise0_prev_r == mux_rd_rise0_r[0]) &
845  (mpr_rd_fall0_prev_r == mux_rd_fall0_r[0]) &
846  (mpr_rd_rise1_prev_r == mux_rd_rise1_r[0]) &
847  (mpr_rd_fall1_prev_r == mux_rd_fall1_r[0]) &
848  (stable_idel_cnt < 3'd2))
849  stable_idel_cnt <= #TCQ stable_idel_cnt + 1;
850  end
851  end
852 
853  always @(posedge clk) begin
854  if (rst |
855  (mpr_rd_rise0_prev_r & ~mpr_rd_fall0_prev_r &
856  mpr_rd_rise1_prev_r & ~mpr_rd_fall1_prev_r))
857  inhibit_edge_detect_r <= 1'b1;
858  else if ((cal1_state_r == CAL1_MPR_PAT_DETECT) &
859  (idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing] > 5'd1) &
860  (~mpr_rd_rise0_prev_r & mpr_rd_fall0_prev_r &
861  ~mpr_rd_rise1_prev_r & mpr_rd_fall1_prev_r))
862  inhibit_edge_detect_r <= 1'b0;
863  end
864 
865  //checking for transition from 01010101 to 10101010
866  always @(posedge clk)begin
867  if (rst | (cal1_state_r == CAL1_MPR_NEW_DQS_WAIT) |
868  inhibit_edge_detect_r)
869  idel_mpr_pat_detect_r <= #TCQ 1'b0;
870  // 1010 is not the correct pattern
871  else if ((mpr_rd_rise0_prev_r & ~mpr_rd_fall0_prev_r &
872  mpr_rd_rise1_prev_r & ~mpr_rd_fall1_prev_r) ||
873  ((stable_idel_cnt < 3'd2) & (cal1_state_r == CAL1_MPR_PAT_DETECT)
874  & (idel_pat_detect_valid_r)))
875  // ||(idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing] < 5'd2))
876  idel_mpr_pat_detect_r <= #TCQ 1'b0;
877  // 0101 to 1010 is the correct transition
878  else if ((~mpr_rd_rise0_prev_r & mpr_rd_fall0_prev_r &
879  ~mpr_rd_rise1_prev_r & mpr_rd_fall1_prev_r) &
880  (stable_idel_cnt == 3'd2) &
881  ((mpr_rd_rise0_prev_r != mux_rd_rise0_r[0]) ||
882  (mpr_rd_fall0_prev_r != mux_rd_fall0_r[0]) ||
883  (mpr_rd_rise1_prev_r != mux_rd_rise1_r[0]) ||
884  (mpr_rd_fall1_prev_r != mux_rd_fall1_r[0])))
885  idel_mpr_pat_detect_r <= #TCQ 1'b1;
886  end
887  end
888  endgenerate
889 
890 
891 
892  // Registered signal indicates when mux_rd_rise/fall_r is valid
893  always @(posedge clk)
894  mux_rd_valid_r <= #TCQ ~phy_if_empty;
895 
896 
897  //***************************************************************************
898  // Decrement initial Phaser_IN fine delay value before proceeding with
899  // read calibration
900  //***************************************************************************
901 
902  always @(posedge clk) begin
903  dqs_po_dec_done_r1 <= #TCQ dqs_po_dec_done;
904  dqs_po_dec_done_r2 <= #TCQ dqs_po_dec_done_r1;
905  fine_dly_dec_done_r2 <= #TCQ fine_dly_dec_done_r1;
906  pi_fine_dly_dec_done <= #TCQ fine_dly_dec_done_r2;
907  end
908 
909  always @(posedge clk) begin
910  if (rst || pi_cnt_dec)
911  wait_cnt_r <= #TCQ 'd8;
912  else if (dqs_po_dec_done_r2 && (wait_cnt_r > 'd0))
913  wait_cnt_r <= #TCQ wait_cnt_r - 1;
914  end
915 
916  always @(posedge clk) begin
917  if (rst) begin
918  pi_rdval_cnt <= #TCQ 'd0;
919  end else if (dqs_po_dec_done_r1 && ~dqs_po_dec_done_r2) begin
920  pi_rdval_cnt <= #TCQ pi_counter_read_val;
921  end else if (pi_rdval_cnt > 'd0) begin
922  if (pi_cnt_dec)
923  pi_rdval_cnt <= #TCQ pi_rdval_cnt - 1;
924  else
925  pi_rdval_cnt <= #TCQ pi_rdval_cnt;
926  end else if (pi_rdval_cnt == 'd0) begin
927  pi_rdval_cnt <= #TCQ pi_rdval_cnt;
928  end
929  end
930 
931  always @(posedge clk) begin
932  if (rst || (pi_rdval_cnt == 'd0))
933  pi_cnt_dec <= #TCQ 1'b0;
934  else if (dqs_po_dec_done_r2 && (pi_rdval_cnt > 'd0)
935  && (wait_cnt_r == 'd1))
936  pi_cnt_dec <= #TCQ 1'b1;
937  else
938  pi_cnt_dec <= #TCQ 1'b0;
939  end
940 
941  always @(posedge clk) begin
942  if (rst) begin
943  fine_dly_dec_done_r1 <= #TCQ 1'b0;
944  end else if (((pi_cnt_dec == 'd1) && (pi_rdval_cnt == 'd1)) ||
945  (dqs_po_dec_done_r2 && (pi_rdval_cnt == 'd0))) begin
946  fine_dly_dec_done_r1 <= #TCQ 1'b1;
947  end
948  end
949 
950  //***************************************************************************
951  // Demultiplexor to control Phaser_IN delay values
952  //***************************************************************************
953 
954  // Read DQS
955  always @(posedge clk) begin
956  if (rst) begin
957  pi_en_stg2_f_timing <= #TCQ 'b0;
958  pi_stg2_f_incdec_timing <= #TCQ 'b0;
959  end else if (pi_cnt_dec) begin
960  pi_en_stg2_f_timing <= #TCQ 'b1;
961  pi_stg2_f_incdec_timing <= #TCQ 'b0;
962  end else if (cal1_dlyce_cpt_r) begin
963  if ((SIM_CAL_OPTION == "NONE") ||
964  (SIM_CAL_OPTION == "FAST_WIN_DETECT")) begin
965  // Change only specified DQS
966  pi_en_stg2_f_timing <= #TCQ 1'b1;
967  pi_stg2_f_incdec_timing <= #TCQ cal1_dlyinc_cpt_r;
968  end else if (SIM_CAL_OPTION == "FAST_CAL") begin
969  // if simulating, and "shortcuts" for calibration enabled, apply
970  // results to all DQSs (i.e. assume same delay on all
971  // DQSs).
972  pi_en_stg2_f_timing <= #TCQ 1'b1;
973  pi_stg2_f_incdec_timing <= #TCQ cal1_dlyinc_cpt_r;
974  end
975  end else begin
976  pi_en_stg2_f_timing <= #TCQ 'b0;
977  pi_stg2_f_incdec_timing <= #TCQ 'b0;
978  end
979  end
980 
981  // registered for timing
982  always @(posedge clk) begin
983  pi_en_stg2_f <= #TCQ pi_en_stg2_f_timing;
984  pi_stg2_f_incdec <= #TCQ pi_stg2_f_incdec_timing;
985  end
986 
987  // This counter used to implement settling time between
988  // Phaser_IN rank register loads to different DQSs
989  always @(posedge clk) begin
990  if (rst)
991  done_cnt <= #TCQ 'b0;
992  else if (((cal1_state_r == CAL1_REGL_LOAD) &&
993  (cal1_state_r1 == CAL1_NEXT_DQS)) ||
994  ((done_cnt == 4'd1) && (cal1_state_r != CAL1_DONE)))
995  done_cnt <= #TCQ 4'b1010;
996  else if (done_cnt > 'b0)
997  done_cnt <= #TCQ done_cnt - 1;
998  end
999 
1000  // During rank register loading the rank count must be sent to
1001  // Phaser_IN via the phy_ctl_wd?? If so phy_init will have to
1002  // issue NOPs during rank register loading with the appropriate
1003  // rank count
1004  always @(posedge clk) begin
1005  if (rst || (regl_rank_done_r == 1'b1))
1006  regl_rank_done_r <= #TCQ 1'b0;
1007  else if ((regl_dqs_cnt == DQS_WIDTH-1) &&
1008  (regl_rank_cnt != RANKS-1) &&
1009  (done_cnt == 4'd1))
1010  regl_rank_done_r <= #TCQ 1'b1;
1011  end
1012 
1013  // Temp wire for timing.
1014  // The following in the always block below causes timing issues
1015  // due to DSP block inference
1016  // 6*regl_dqs_cnt.
1017  // replacing this with two left shifts + 1 left shift to avoid
1018  // DSP multiplier.
1019  assign regl_dqs_cnt_timing = {2'd0, regl_dqs_cnt};
1020 
1021  // Load Phaser_OUT rank register with rdlvl delay value
1022  // for each DQS per rank.
1023  always @(posedge clk) begin
1024  if (rst || (done_cnt == 4'd0)) begin
1025  pi_stg2_load_timing <= #TCQ 'b0;
1026  pi_stg2_reg_l_timing <= #TCQ 'b0;
1027  end else if ((cal1_state_r == CAL1_REGL_LOAD) &&
1028  (regl_dqs_cnt <= DQS_WIDTH-1) && (done_cnt == 4'd1)) begin
1029  pi_stg2_load_timing <= #TCQ 'b1;
1030  pi_stg2_reg_l_timing <= #TCQ
1031  rdlvl_dqs_tap_cnt_r[rnk_cnt_r][regl_dqs_cnt];
1032  end else begin
1033  pi_stg2_load_timing <= #TCQ 'b0;
1034  pi_stg2_reg_l_timing <= #TCQ 'b0;
1035  end
1036  end
1037 
1038  // registered for timing
1039  always @(posedge clk) begin
1040  pi_stg2_load <= #TCQ pi_stg2_load_timing;
1041  pi_stg2_reg_l <= #TCQ pi_stg2_reg_l_timing;
1042  end
1043 
1044  always @(posedge clk) begin
1045  if (rst || (done_cnt == 4'd0) ||
1046  (mpr_rdlvl_done_r1 && ~mpr_rdlvl_done_r2))
1047  regl_rank_cnt <= #TCQ 2'b00;
1048  else if ((cal1_state_r == CAL1_REGL_LOAD) &&
1049  (regl_dqs_cnt == DQS_WIDTH-1) && (done_cnt == 4'd1)) begin
1050  if (regl_rank_cnt == RANKS-1)
1051  regl_rank_cnt <= #TCQ regl_rank_cnt;
1052  else
1053  regl_rank_cnt <= #TCQ regl_rank_cnt + 1;
1054  end
1055  end
1056 
1057  always @(posedge clk) begin
1058  if (rst || (done_cnt == 4'd0) ||
1059  (mpr_rdlvl_done_r1 && ~mpr_rdlvl_done_r2))
1060  regl_dqs_cnt <= #TCQ {DQS_CNT_WIDTH+1{1'b0}};
1061  else if ((cal1_state_r == CAL1_REGL_LOAD) &&
1062  (regl_dqs_cnt == DQS_WIDTH-1) && (done_cnt == 4'd1)) begin
1063  if (regl_rank_cnt == RANKS-1)
1064  regl_dqs_cnt <= #TCQ regl_dqs_cnt;
1065  else
1066  regl_dqs_cnt <= #TCQ 'b0;
1067  end else if ((cal1_state_r == CAL1_REGL_LOAD) && (regl_dqs_cnt != DQS_WIDTH-1)
1068  && (done_cnt == 4'd1))
1069  regl_dqs_cnt <= #TCQ regl_dqs_cnt + 1;
1070  else
1071  regl_dqs_cnt <= #TCQ regl_dqs_cnt;
1072  end
1073 
1074 
1075  always @(posedge clk)
1076  regl_dqs_cnt_r <= #TCQ regl_dqs_cnt;
1077  //*****************************************************************
1078  // DQ Stage 1 CALIBRATION INCREMENT/DECREMENT LOGIC:
1079  // The actual IDELAY elements for each of the DQ bits is set via the
1080  // DLYVAL parallel load port. However, the stage 1 calibration
1081  // algorithm (well most of it) only needs to increment or decrement the DQ
1082  // IDELAY value by 1 at any one time.
1083  //*****************************************************************
1084 
1085  // Chip-select generation for each of the individual counters tracking
1086  // IDELAY tap values for each DQ
1087  generate
1088  for (z = 0; z < DQS_WIDTH; z = z + 1) begin: gen_dlyce_dq
1089  always @(posedge clk)
1090  if (rst)
1091  dlyce_dq_r[DRAM_WIDTH*z+:DRAM_WIDTH] <= #TCQ 'b0;
1092  else
1093  if (SIM_CAL_OPTION == "SKIP_CAL")
1094  // If skipping calibration altogether (only for simulation), no
1095  // need to set DQ IODELAY values - they are hardcoded
1096  dlyce_dq_r[DRAM_WIDTH*z+:DRAM_WIDTH] <= #TCQ 'b0;
1097  else if (SIM_CAL_OPTION == "FAST_CAL") begin
1098  // If fast calibration option (simulation only) selected, DQ
1099  // IODELAYs across all bytes are updated simultaneously
1100  // (although per-bit deskew within DQS[0] is still supported)
1101  for (h = 0; h < DRAM_WIDTH; h = h + 1) begin
1102  dlyce_dq_r[DRAM_WIDTH*z + h] <= #TCQ cal1_dlyce_dq_r;
1103  end
1104  end else if ((SIM_CAL_OPTION == "NONE") ||
1105  (SIM_CAL_OPTION == "FAST_WIN_DETECT")) begin
1106  if (cal1_cnt_cpt_r == z) begin
1107  for (g = 0; g < DRAM_WIDTH; g = g + 1) begin
1108  dlyce_dq_r[DRAM_WIDTH*z + g]
1109  <= #TCQ cal1_dlyce_dq_r;
1110  end
1111  end else
1112  dlyce_dq_r[DRAM_WIDTH*z+:DRAM_WIDTH] <= #TCQ 'b0;
1113  end
1114  end
1115  endgenerate
1116 
1117  // Also delay increment/decrement control to match delay on DLYCE
1118  always @(posedge clk)
1119  if (rst)
1120  dlyinc_dq_r <= #TCQ 1'b0;
1121  else
1122  dlyinc_dq_r <= #TCQ cal1_dlyinc_dq_r;
1123 
1124 
1125  // Each DQ has a counter associated with it to record current read-leveling
1126  // delay value
1127  always @(posedge clk)
1128  // Reset or skipping calibration all together
1129  if (rst | (SIM_CAL_OPTION == "SKIP_CAL")) begin
1130  for (aa = 0; aa < RANKS; aa = aa + 1) begin: rst_dlyval_dq_reg_r
1131  for (bb = 0; bb < DQ_WIDTH; bb = bb + 1)
1132  dlyval_dq_reg_r[aa][bb] <= #TCQ 'b0;
1133  end
1134  end else if (SIM_CAL_OPTION == "FAST_CAL") begin
1135  for (n = 0; n < RANKS; n = n + 1) begin: gen_dlyval_dq_reg_rnk
1136  for (r = 0; r < DQ_WIDTH; r = r + 1) begin: gen_dlyval_dq_reg
1137  if (dlyce_dq_r[r]) begin
1138  if (dlyinc_dq_r)
1139  dlyval_dq_reg_r[n][r] <= #TCQ dlyval_dq_reg_r[n][r] + 5'h01;
1140  else
1141  dlyval_dq_reg_r[n][r] <= #TCQ dlyval_dq_reg_r[n][r] - 5'h01;
1142  end
1143  end
1144  end
1145  end else begin
1146  if (dlyce_dq_r[cal1_cnt_cpt_r]) begin
1147  if (dlyinc_dq_r)
1148  dlyval_dq_reg_r[rnk_cnt_r][cal1_cnt_cpt_r] <= #TCQ
1149  dlyval_dq_reg_r[rnk_cnt_r][cal1_cnt_cpt_r] + 5'h01;
1150  else
1151  dlyval_dq_reg_r[rnk_cnt_r][cal1_cnt_cpt_r] <= #TCQ
1152  dlyval_dq_reg_r[rnk_cnt_r][cal1_cnt_cpt_r] - 5'h01;
1153  end
1154  end
1155 
1156  // Register for timing (help with logic placement)
1157  always @(posedge clk) begin
1158  for (cc = 0; cc < RANKS; cc = cc + 1) begin: dlyval_dq_assgn
1159  for (dd = 0; dd < DQ_WIDTH; dd = dd + 1)
1160  dlyval_dq[((5*dd)+(cc*DQ_WIDTH*5))+:5] <= #TCQ dlyval_dq_reg_r[cc][dd];
1161  end
1162  end
1163 
1164  //***************************************************************************
1165  // Generate signal used to delay calibration state machine - used when:
1166  // (1) IDELAY value changed
1167  // (2) RD_MUX_SEL value changed
1168  // Use when a delay is necessary to give the change time to propagate
1169  // through the data pipeline (through IDELAY and ISERDES, and fabric
1170  // pipeline stages)
1171  //***************************************************************************
1172 
1173 
1174  // List all the stage 1 calibration wait states here.
1175  always @(posedge clk)
1176  if ((cal1_state_r == CAL1_NEW_DQS_WAIT) ||
1177  (cal1_state_r == CAL1_MPR_NEW_DQS_WAIT) ||
1178  (cal1_state_r == CAL1_NEW_DQS_PREWAIT) ||
1179  (cal1_state_r == CAL1_VALID_WAIT) ||
1180  (cal1_state_r == CAL1_PB_STORE_FIRST_WAIT) ||
1181  (cal1_state_r == CAL1_PB_INC_CPT_WAIT) ||
1182  (cal1_state_r == CAL1_PB_DEC_CPT_LEFT_WAIT) ||
1183  (cal1_state_r == CAL1_PB_INC_DQ_WAIT) ||
1184  (cal1_state_r == CAL1_PB_DEC_CPT_WAIT) ||
1185  (cal1_state_r == CAL1_IDEL_INC_CPT_WAIT) ||
1186  (cal1_state_r == CAL1_IDEL_DEC_CPT_WAIT) ||
1187  (cal1_state_r == CAL1_STORE_FIRST_WAIT) ||
1188  (cal1_state_r == CAL1_DQ_IDEL_TAP_INC_WAIT) ||
1189  (cal1_state_r == CAL1_DQ_IDEL_TAP_DEC_WAIT))
1190  cal1_wait_cnt_en_r <= #TCQ 1'b1;
1191  else
1192  cal1_wait_cnt_en_r <= #TCQ 1'b0;
1193 
1194  always @(posedge clk)
1195  if (!cal1_wait_cnt_en_r) begin
1196  cal1_wait_cnt_r <= #TCQ 5'b00000;
1197  cal1_wait_r <= #TCQ 1'b1;
1198  end else begin
1199  if (cal1_wait_cnt_r != PIPE_WAIT_CNT - 1) begin
1200  cal1_wait_cnt_r <= #TCQ cal1_wait_cnt_r + 1;
1201  cal1_wait_r <= #TCQ 1'b1;
1202  end else begin
1203  // Need to reset to 0 to handle the case when there are two
1204  // different WAIT states back-to-back
1205  cal1_wait_cnt_r <= #TCQ 5'b00000;
1206  cal1_wait_r <= #TCQ 1'b0;
1207  end
1208  end
1209 
1210  //***************************************************************************
1211  // generate request to PHY_INIT logic to issue precharged. Required when
1212  // calibration can take a long time (during which there are only constant
1213  // reads present on this bus). In this case need to issue perioidic
1214  // precharges to avoid tRAS violation. This signal must meet the following
1215  // requirements: (1) only transition from 0->1 when prech is first needed,
1216  // (2) stay at 1 and only transition 1->0 when RDLVL_PRECH_DONE asserted
1217  //***************************************************************************
1218 
1219  always @(posedge clk)
1220  if (rst)
1221  rdlvl_prech_req <= #TCQ 1'b0;
1222  else
1223  rdlvl_prech_req <= #TCQ cal1_prech_req_r;
1224 
1225  //***************************************************************************
1226  // Serial-to-parallel register to store last RDDATA_SHIFT_LEN cycles of
1227  // data from ISERDES. The value of this register is also stored, so that
1228  // previous and current values of the ISERDES data can be compared while
1229  // varying the IODELAY taps to see if an "edge" of the data valid window
1230  // has been encountered since the last IODELAY tap adjustment
1231  //***************************************************************************
1232 
1233  //***************************************************************************
1234  // Shift register to store last RDDATA_SHIFT_LEN cycles of data from ISERDES
1235  // NOTE: Written using discrete flops, but SRL can be used if the matching
1236  // logic does the comparison sequentially, rather than parallel
1237  //***************************************************************************
1238 
1239  generate
1240  genvar rd_i;
1241  if (nCK_PER_CLK == 4) begin: gen_sr_div4
1242  if (RD_SHIFT_LEN == 1) begin: gen_sr_len_eq1
1243  for (rd_i = 0; rd_i < DRAM_WIDTH; rd_i = rd_i + 1) begin: gen_sr
1244  always @(posedge clk) begin
1245  if (mux_rd_valid_r) begin
1246  sr_rise0_r[rd_i] <= #TCQ mux_rd_rise0_r[rd_i];
1247  sr_fall0_r[rd_i] <= #TCQ mux_rd_fall0_r[rd_i];
1248  sr_rise1_r[rd_i] <= #TCQ mux_rd_rise1_r[rd_i];
1249  sr_fall1_r[rd_i] <= #TCQ mux_rd_fall1_r[rd_i];
1250  sr_rise2_r[rd_i] <= #TCQ mux_rd_rise2_r[rd_i];
1251  sr_fall2_r[rd_i] <= #TCQ mux_rd_fall2_r[rd_i];
1252  sr_rise3_r[rd_i] <= #TCQ mux_rd_rise3_r[rd_i];
1253  sr_fall3_r[rd_i] <= #TCQ mux_rd_fall3_r[rd_i];
1254  end
1255  end
1256  end
1257  end else if (RD_SHIFT_LEN > 1) begin: gen_sr_len_gt1
1258  for (rd_i = 0; rd_i < DRAM_WIDTH; rd_i = rd_i + 1) begin: gen_sr
1259  always @(posedge clk) begin
1260  if (mux_rd_valid_r) begin
1261  sr_rise0_r[rd_i] <= #TCQ {sr_rise0_r[rd_i][RD_SHIFT_LEN-2:0],
1262  mux_rd_rise0_r[rd_i]};
1263  sr_fall0_r[rd_i] <= #TCQ {sr_fall0_r[rd_i][RD_SHIFT_LEN-2:0],
1264  mux_rd_fall0_r[rd_i]};
1265  sr_rise1_r[rd_i] <= #TCQ {sr_rise1_r[rd_i][RD_SHIFT_LEN-2:0],
1266  mux_rd_rise1_r[rd_i]};
1267  sr_fall1_r[rd_i] <= #TCQ {sr_fall1_r[rd_i][RD_SHIFT_LEN-2:0],
1268  mux_rd_fall1_r[rd_i]};
1269  sr_rise2_r[rd_i] <= #TCQ {sr_rise2_r[rd_i][RD_SHIFT_LEN-2:0],
1270  mux_rd_rise2_r[rd_i]};
1271  sr_fall2_r[rd_i] <= #TCQ {sr_fall2_r[rd_i][RD_SHIFT_LEN-2:0],
1272  mux_rd_fall2_r[rd_i]};
1273  sr_rise3_r[rd_i] <= #TCQ {sr_rise3_r[rd_i][RD_SHIFT_LEN-2:0],
1274  mux_rd_rise3_r[rd_i]};
1275  sr_fall3_r[rd_i] <= #TCQ {sr_fall3_r[rd_i][RD_SHIFT_LEN-2:0],
1276  mux_rd_fall3_r[rd_i]};
1277  end
1278  end
1279  end
1280  end
1281  end else if (nCK_PER_CLK == 2) begin: gen_sr_div2
1282  if (RD_SHIFT_LEN == 1) begin: gen_sr_len_eq1
1283  for (rd_i = 0; rd_i < DRAM_WIDTH; rd_i = rd_i + 1) begin: gen_sr
1284  always @(posedge clk) begin
1285  if (mux_rd_valid_r) begin
1286  sr_rise0_r[rd_i] <= #TCQ {mux_rd_rise0_r[rd_i]};
1287  sr_fall0_r[rd_i] <= #TCQ {mux_rd_fall0_r[rd_i]};
1288  sr_rise1_r[rd_i] <= #TCQ {mux_rd_rise1_r[rd_i]};
1289  sr_fall1_r[rd_i] <= #TCQ {mux_rd_fall1_r[rd_i]};
1290  end
1291  end
1292  end
1293  end else if (RD_SHIFT_LEN > 1) begin: gen_sr_len_gt1
1294  for (rd_i = 0; rd_i < DRAM_WIDTH; rd_i = rd_i + 1) begin: gen_sr
1295  always @(posedge clk) begin
1296  if (mux_rd_valid_r) begin
1297  sr_rise0_r[rd_i] <= #TCQ {sr_rise0_r[rd_i][RD_SHIFT_LEN-2:0],
1298  mux_rd_rise0_r[rd_i]};
1299  sr_fall0_r[rd_i] <= #TCQ {sr_fall0_r[rd_i][RD_SHIFT_LEN-2:0],
1300  mux_rd_fall0_r[rd_i]};
1301  sr_rise1_r[rd_i] <= #TCQ {sr_rise1_r[rd_i][RD_SHIFT_LEN-2:0],
1302  mux_rd_rise1_r[rd_i]};
1303  sr_fall1_r[rd_i] <= #TCQ {sr_fall1_r[rd_i][RD_SHIFT_LEN-2:0],
1304  mux_rd_fall1_r[rd_i]};
1305  end
1306  end
1307  end
1308  end
1309  end
1310  endgenerate
1311 
1312  //***************************************************************************
1313  // Conversion to pattern calibration
1314  //***************************************************************************
1315 
1316  // Pattern for DQ IDELAY calibration
1317 
1318  //*****************************************************************
1319  // Expected data pattern when DQ shifted to the right such that
1320  // DQS before the left edge of the DVW:
1321  // Based on pattern of ({rise,fall}) =
1322  // 0x1, 0xB, 0x4, 0x4, 0xB, 0x9
1323  // Each nibble will look like:
1324  // bit3: 0, 1, 0, 0, 1, 1
1325  // bit2: 0, 0, 1, 1, 0, 0
1326  // bit1: 0, 1, 0, 0, 1, 0
1327  // bit0: 1, 1, 0, 0, 1, 1
1328  // Or if the write is early it could look like:
1329  // 0x4, 0x4, 0xB, 0x9, 0x6, 0xE
1330  // bit3: 0, 0, 1, 1, 0, 1
1331  // bit2: 1, 1, 0, 0, 1, 1
1332  // bit1: 0, 0, 1, 0, 1, 1
1333  // bit0: 0, 0, 1, 1, 0, 0
1334  // Change the hard-coded pattern below accordingly as RD_SHIFT_LEN
1335  // and the actual training pattern contents change
1336  //*****************************************************************
1337 
1338  generate
1339  if (nCK_PER_CLK == 4) begin: gen_pat_div4
1340  // Pattern for DQ IDELAY increment
1341 
1342  // Target pattern for "early write"
1343  assign {idel_pat0_rise0[3], idel_pat0_rise0[2],
1344  idel_pat0_rise0[1], idel_pat0_rise0[0]} = 4'h1;
1345  assign {idel_pat0_fall0[3], idel_pat0_fall0[2],
1346  idel_pat0_fall0[1], idel_pat0_fall0[0]} = 4'h7;
1347  assign {idel_pat0_rise1[3], idel_pat0_rise1[2],
1348  idel_pat0_rise1[1], idel_pat0_rise1[0]} = 4'hE;
1349  assign {idel_pat0_fall1[3], idel_pat0_fall1[2],
1350  idel_pat0_fall1[1], idel_pat0_fall1[0]} = 4'hC;
1351  assign {idel_pat0_rise2[3], idel_pat0_rise2[2],
1352  idel_pat0_rise2[1], idel_pat0_rise2[0]} = 4'h9;
1353  assign {idel_pat0_fall2[3], idel_pat0_fall2[2],
1354  idel_pat0_fall2[1], idel_pat0_fall2[0]} = 4'h2;
1355  assign {idel_pat0_rise3[3], idel_pat0_rise3[2],
1356  idel_pat0_rise3[1], idel_pat0_rise3[0]} = 4'h4;
1357  assign {idel_pat0_fall3[3], idel_pat0_fall3[2],
1358  idel_pat0_fall3[1], idel_pat0_fall3[0]} = 4'hB;
1359 
1360  // Target pattern for "on-time write"
1361  assign {idel_pat1_rise0[3], idel_pat1_rise0[2],
1362  idel_pat1_rise0[1], idel_pat1_rise0[0]} = 4'h4;
1363  assign {idel_pat1_fall0[3], idel_pat1_fall0[2],
1364  idel_pat1_fall0[1], idel_pat1_fall0[0]} = 4'h9;
1365  assign {idel_pat1_rise1[3], idel_pat1_rise1[2],
1366  idel_pat1_rise1[1], idel_pat1_rise1[0]} = 4'h3;
1367  assign {idel_pat1_fall1[3], idel_pat1_fall1[2],
1368  idel_pat1_fall1[1], idel_pat1_fall1[0]} = 4'h7;
1369  assign {idel_pat1_rise2[3], idel_pat1_rise2[2],
1370  idel_pat1_rise2[1], idel_pat1_rise2[0]} = 4'hE;
1371  assign {idel_pat1_fall2[3], idel_pat1_fall2[2],
1372  idel_pat1_fall2[1], idel_pat1_fall2[0]} = 4'hC;
1373  assign {idel_pat1_rise3[3], idel_pat1_rise3[2],
1374  idel_pat1_rise3[1], idel_pat1_rise3[0]} = 4'h9;
1375  assign {idel_pat1_fall3[3], idel_pat1_fall3[2],
1376  idel_pat1_fall3[1], idel_pat1_fall3[0]} = 4'h2;
1377 
1378 
1379  // Correct data valid window for "early write"
1380  assign {pat0_rise0[3], pat0_rise0[2],
1381  pat0_rise0[1], pat0_rise0[0]} = 4'h7;
1382  assign {pat0_fall0[3], pat0_fall0[2],
1383  pat0_fall0[1], pat0_fall0[0]} = 4'hE;
1384  assign {pat0_rise1[3], pat0_rise1[2],
1385  pat0_rise1[1], pat0_rise1[0]} = 4'hC;
1386  assign {pat0_fall1[3], pat0_fall1[2],
1387  pat0_fall1[1], pat0_fall1[0]} = 4'h9;
1388  assign {pat0_rise2[3], pat0_rise2[2],
1389  pat0_rise2[1], pat0_rise2[0]} = 4'h2;
1390  assign {pat0_fall2[3], pat0_fall2[2],
1391  pat0_fall2[1], pat0_fall2[0]} = 4'h4;
1392  assign {pat0_rise3[3], pat0_rise3[2],
1393  pat0_rise3[1], pat0_rise3[0]} = 4'hB;
1394  assign {pat0_fall3[3], pat0_fall3[2],
1395  pat0_fall3[1], pat0_fall3[0]} = 4'h1;
1396 
1397  // Correct data valid window for "on-time write"
1398  assign {pat1_rise0[3], pat1_rise0[2],
1399  pat1_rise0[1], pat1_rise0[0]} = 4'h9;
1400  assign {pat1_fall0[3], pat1_fall0[2],
1401  pat1_fall0[1], pat1_fall0[0]} = 4'h3;
1402  assign {pat1_rise1[3], pat1_rise1[2],
1403  pat1_rise1[1], pat1_rise1[0]} = 4'h7;
1404  assign {pat1_fall1[3], pat1_fall1[2],
1405  pat1_fall1[1], pat1_fall1[0]} = 4'hE;
1406  assign {pat1_rise2[3], pat1_rise2[2],
1407  pat1_rise2[1], pat1_rise2[0]} = 4'hC;
1408  assign {pat1_fall2[3], pat1_fall2[2],
1409  pat1_fall2[1], pat1_fall2[0]} = 4'h9;
1410  assign {pat1_rise3[3], pat1_rise3[2],
1411  pat1_rise3[1], pat1_rise3[0]} = 4'h2;
1412  assign {pat1_fall3[3], pat1_fall3[2],
1413  pat1_fall3[1], pat1_fall3[0]} = 4'h4;
1414 
1415  end else if (nCK_PER_CLK == 2) begin: gen_pat_div2
1416 
1417  // Pattern for DQ IDELAY increment
1418 
1419  // Target pattern for "early write"
1420  assign idel_pat0_rise0[3] = 2'b01;
1421  assign idel_pat0_fall0[3] = 2'b00;
1422  assign idel_pat0_rise1[3] = 2'b10;
1423  assign idel_pat0_fall1[3] = 2'b11;
1424 
1425  assign idel_pat0_rise0[2] = 2'b00;
1426  assign idel_pat0_fall0[2] = 2'b10;
1427  assign idel_pat0_rise1[2] = 2'b11;
1428  assign idel_pat0_fall1[2] = 2'b10;
1429 
1430  assign idel_pat0_rise0[1] = 2'b00;
1431  assign idel_pat0_fall0[1] = 2'b11;
1432  assign idel_pat0_rise1[1] = 2'b10;
1433  assign idel_pat0_fall1[1] = 2'b01;
1434 
1435  assign idel_pat0_rise0[0] = 2'b11;
1436  assign idel_pat0_fall0[0] = 2'b10;
1437  assign idel_pat0_rise1[0] = 2'b00;
1438  assign idel_pat0_fall1[0] = 2'b01;
1439 
1440 
1441  // Target pattern for "on-time write"
1442  assign idel_pat1_rise0[3] = 2'b01;
1443  assign idel_pat1_fall0[3] = 2'b11;
1444  assign idel_pat1_rise1[3] = 2'b01;
1445  assign idel_pat1_fall1[3] = 2'b00;
1446 
1447  assign idel_pat1_rise0[2] = 2'b11;
1448  assign idel_pat1_fall0[2] = 2'b01;
1449  assign idel_pat1_rise1[2] = 2'b00;
1450  assign idel_pat1_fall1[2] = 2'b10;
1451 
1452  assign idel_pat1_rise0[1] = 2'b01;
1453  assign idel_pat1_fall0[1] = 2'b00;
1454  assign idel_pat1_rise1[1] = 2'b10;
1455  assign idel_pat1_fall1[1] = 2'b11;
1456 
1457  assign idel_pat1_rise0[0] = 2'b00;
1458  assign idel_pat1_fall0[0] = 2'b10;
1459  assign idel_pat1_rise1[0] = 2'b11;
1460  assign idel_pat1_fall1[0] = 2'b10;
1461 
1462 
1463  // Correct data valid window for "early write"
1464  assign pat0_rise0[3] = 2'b00;
1465  assign pat0_fall0[3] = 2'b10;
1466  assign pat0_rise1[3] = 2'b11;
1467  assign pat0_fall1[3] = 2'b10;
1468 
1469  assign pat0_rise0[2] = 2'b10;
1470  assign pat0_fall0[2] = 2'b11;
1471  assign pat0_rise1[2] = 2'b10;
1472  assign pat0_fall1[2] = 2'b00;
1473 
1474  assign pat0_rise0[1] = 2'b11;
1475  assign pat0_fall0[1] = 2'b10;
1476  assign pat0_rise1[1] = 2'b01;
1477  assign pat0_fall1[1] = 2'b00;
1478 
1479  assign pat0_rise0[0] = 2'b10;
1480  assign pat0_fall0[0] = 2'b00;
1481  assign pat0_rise1[0] = 2'b01;
1482  assign pat0_fall1[0] = 2'b11;
1483 
1484  // Correct data valid window for "on-time write"
1485  assign pat1_rise0[3] = 2'b11;
1486  assign pat1_fall0[3] = 2'b01;
1487  assign pat1_rise1[3] = 2'b00;
1488  assign pat1_fall1[3] = 2'b10;
1489 
1490  assign pat1_rise0[2] = 2'b01;
1491  assign pat1_fall0[2] = 2'b00;
1492  assign pat1_rise1[2] = 2'b10;
1493  assign pat1_fall1[2] = 2'b11;
1494 
1495  assign pat1_rise0[1] = 2'b00;
1496  assign pat1_fall0[1] = 2'b10;
1497  assign pat1_rise1[1] = 2'b11;
1498  assign pat1_fall1[1] = 2'b10;
1499 
1500  assign pat1_rise0[0] = 2'b10;
1501  assign pat1_fall0[0] = 2'b11;
1502  assign pat1_rise1[0] = 2'b10;
1503  assign pat1_fall1[0] = 2'b00;
1504  end
1505  endgenerate
1506 
1507  // Each bit of each byte is compared to expected pattern.
1508  // This was done to prevent (and "drastically decrease") the chance that
1509  // invalid data clocked in when the DQ bus is tri-state (along with a
1510  // combination of the correct data) will resemble the expected data
1511  // pattern. A better fix for this is to change the training pattern and/or
1512  // make the pattern longer.
1513  generate
1514  genvar pt_i;
1515  if (nCK_PER_CLK == 4) begin: gen_pat_match_div4
1516  for (pt_i = 0; pt_i < DRAM_WIDTH; pt_i = pt_i + 1) begin: gen_pat_match
1517 
1518  // DQ IDELAY pattern detection
1519  always @(posedge clk) begin
1520  if (sr_rise0_r[pt_i] == idel_pat0_rise0[pt_i%4])
1521  idel_pat0_match_rise0_r[pt_i] <= #TCQ 1'b1;
1522  else
1523  idel_pat0_match_rise0_r[pt_i] <= #TCQ 1'b0;
1524 
1525  if (sr_fall0_r[pt_i] == idel_pat0_fall0[pt_i%4])
1526  idel_pat0_match_fall0_r[pt_i] <= #TCQ 1'b1;
1527  else
1528  idel_pat0_match_fall0_r[pt_i] <= #TCQ 1'b0;
1529 
1530  if (sr_rise1_r[pt_i] == idel_pat0_rise1[pt_i%4])
1531  idel_pat0_match_rise1_r[pt_i] <= #TCQ 1'b1;
1532  else
1533  idel_pat0_match_rise1_r[pt_i] <= #TCQ 1'b0;
1534 
1535  if (sr_fall1_r[pt_i] == idel_pat0_fall1[pt_i%4])
1536  idel_pat0_match_fall1_r[pt_i] <= #TCQ 1'b1;
1537  else
1538  idel_pat0_match_fall1_r[pt_i] <= #TCQ 1'b0;
1539 
1540  if (sr_rise2_r[pt_i] == idel_pat0_rise2[pt_i%4])
1541  idel_pat0_match_rise2_r[pt_i] <= #TCQ 1'b1;
1542  else
1543  idel_pat0_match_rise2_r[pt_i] <= #TCQ 1'b0;
1544 
1545  if (sr_fall2_r[pt_i] == idel_pat0_fall2[pt_i%4])
1546  idel_pat0_match_fall2_r[pt_i] <= #TCQ 1'b1;
1547  else
1548  idel_pat0_match_fall2_r[pt_i] <= #TCQ 1'b0;
1549 
1550  if (sr_rise3_r[pt_i] == idel_pat0_rise3[pt_i%4])
1551  idel_pat0_match_rise3_r[pt_i] <= #TCQ 1'b1;
1552  else
1553  idel_pat0_match_rise3_r[pt_i] <= #TCQ 1'b0;
1554 
1555  if (sr_fall3_r[pt_i] == idel_pat0_fall3[pt_i%4])
1556  idel_pat0_match_fall3_r[pt_i] <= #TCQ 1'b1;
1557  else
1558  idel_pat0_match_fall3_r[pt_i] <= #TCQ 1'b0;
1559  end
1560 
1561  always @(posedge clk) begin
1562  if (sr_rise0_r[pt_i] == idel_pat1_rise0[pt_i%4])
1563  idel_pat1_match_rise0_r[pt_i] <= #TCQ 1'b1;
1564  else
1565  idel_pat1_match_rise0_r[pt_i] <= #TCQ 1'b0;
1566 
1567  if (sr_fall0_r[pt_i] == idel_pat1_fall0[pt_i%4])
1568  idel_pat1_match_fall0_r[pt_i] <= #TCQ 1'b1;
1569  else
1570  idel_pat1_match_fall0_r[pt_i] <= #TCQ 1'b0;
1571 
1572  if (sr_rise1_r[pt_i] == idel_pat1_rise1[pt_i%4])
1573  idel_pat1_match_rise1_r[pt_i] <= #TCQ 1'b1;
1574  else
1575  idel_pat1_match_rise1_r[pt_i] <= #TCQ 1'b0;
1576 
1577  if (sr_fall1_r[pt_i] == idel_pat1_fall1[pt_i%4])
1578  idel_pat1_match_fall1_r[pt_i] <= #TCQ 1'b1;
1579  else
1580  idel_pat1_match_fall1_r[pt_i] <= #TCQ 1'b0;
1581 
1582  if (sr_rise2_r[pt_i] == idel_pat1_rise2[pt_i%4])
1583  idel_pat1_match_rise2_r[pt_i] <= #TCQ 1'b1;
1584  else
1585  idel_pat1_match_rise2_r[pt_i] <= #TCQ 1'b0;
1586 
1587  if (sr_fall2_r[pt_i] == idel_pat1_fall2[pt_i%4])
1588  idel_pat1_match_fall2_r[pt_i] <= #TCQ 1'b1;
1589  else
1590  idel_pat1_match_fall2_r[pt_i] <= #TCQ 1'b0;
1591 
1592  if (sr_rise3_r[pt_i] == idel_pat1_rise3[pt_i%4])
1593  idel_pat1_match_rise3_r[pt_i] <= #TCQ 1'b1;
1594  else
1595  idel_pat1_match_rise3_r[pt_i] <= #TCQ 1'b0;
1596 
1597  if (sr_fall3_r[pt_i] == idel_pat1_fall3[pt_i%4])
1598  idel_pat1_match_fall3_r[pt_i] <= #TCQ 1'b1;
1599  else
1600  idel_pat1_match_fall3_r[pt_i] <= #TCQ 1'b0;
1601  end
1602 
1603  // DQS DVW pattern detection
1604  always @(posedge clk) begin
1605  if (sr_rise0_r[pt_i] == pat0_rise0[pt_i%4])
1606  pat0_match_rise0_r[pt_i] <= #TCQ 1'b1;
1607  else
1608  pat0_match_rise0_r[pt_i] <= #TCQ 1'b0;
1609 
1610  if (sr_fall0_r[pt_i] == pat0_fall0[pt_i%4])
1611  pat0_match_fall0_r[pt_i] <= #TCQ 1'b1;
1612  else
1613  pat0_match_fall0_r[pt_i] <= #TCQ 1'b0;
1614 
1615  if (sr_rise1_r[pt_i] == pat0_rise1[pt_i%4])
1616  pat0_match_rise1_r[pt_i] <= #TCQ 1'b1;
1617  else
1618  pat0_match_rise1_r[pt_i] <= #TCQ 1'b0;
1619 
1620  if (sr_fall1_r[pt_i] == pat0_fall1[pt_i%4])
1621  pat0_match_fall1_r[pt_i] <= #TCQ 1'b1;
1622  else
1623  pat0_match_fall1_r[pt_i] <= #TCQ 1'b0;
1624 
1625  if (sr_rise2_r[pt_i] == pat0_rise2[pt_i%4])
1626  pat0_match_rise2_r[pt_i] <= #TCQ 1'b1;
1627  else
1628  pat0_match_rise2_r[pt_i] <= #TCQ 1'b0;
1629 
1630  if (sr_fall2_r[pt_i] == pat0_fall2[pt_i%4])
1631  pat0_match_fall2_r[pt_i] <= #TCQ 1'b1;
1632  else
1633  pat0_match_fall2_r[pt_i] <= #TCQ 1'b0;
1634 
1635  if (sr_rise3_r[pt_i] == pat0_rise3[pt_i%4])
1636  pat0_match_rise3_r[pt_i] <= #TCQ 1'b1;
1637  else
1638  pat0_match_rise3_r[pt_i] <= #TCQ 1'b0;
1639 
1640  if (sr_fall3_r[pt_i] == pat0_fall3[pt_i%4])
1641  pat0_match_fall3_r[pt_i] <= #TCQ 1'b1;
1642  else
1643  pat0_match_fall3_r[pt_i] <= #TCQ 1'b0;
1644  end
1645 
1646  always @(posedge clk) begin
1647  if (sr_rise0_r[pt_i] == pat1_rise0[pt_i%4])
1648  pat1_match_rise0_r[pt_i] <= #TCQ 1'b1;
1649  else
1650  pat1_match_rise0_r[pt_i] <= #TCQ 1'b0;
1651 
1652  if (sr_fall0_r[pt_i] == pat1_fall0[pt_i%4])
1653  pat1_match_fall0_r[pt_i] <= #TCQ 1'b1;
1654  else
1655  pat1_match_fall0_r[pt_i] <= #TCQ 1'b0;
1656 
1657  if (sr_rise1_r[pt_i] == pat1_rise1[pt_i%4])
1658  pat1_match_rise1_r[pt_i] <= #TCQ 1'b1;
1659  else
1660  pat1_match_rise1_r[pt_i] <= #TCQ 1'b0;
1661 
1662  if (sr_fall1_r[pt_i] == pat1_fall1[pt_i%4])
1663  pat1_match_fall1_r[pt_i] <= #TCQ 1'b1;
1664  else
1665  pat1_match_fall1_r[pt_i] <= #TCQ 1'b0;
1666 
1667  if (sr_rise2_r[pt_i] == pat1_rise2[pt_i%4])
1668  pat1_match_rise2_r[pt_i] <= #TCQ 1'b1;
1669  else
1670  pat1_match_rise2_r[pt_i] <= #TCQ 1'b0;
1671 
1672  if (sr_fall2_r[pt_i] == pat1_fall2[pt_i%4])
1673  pat1_match_fall2_r[pt_i] <= #TCQ 1'b1;
1674  else
1675  pat1_match_fall2_r[pt_i] <= #TCQ 1'b0;
1676 
1677  if (sr_rise3_r[pt_i] == pat1_rise3[pt_i%4])
1678  pat1_match_rise3_r[pt_i] <= #TCQ 1'b1;
1679  else
1680  pat1_match_rise3_r[pt_i] <= #TCQ 1'b0;
1681 
1682  if (sr_fall3_r[pt_i] == pat1_fall3[pt_i%4])
1683  pat1_match_fall3_r[pt_i] <= #TCQ 1'b1;
1684  else
1685  pat1_match_fall3_r[pt_i] <= #TCQ 1'b0;
1686  end
1687 
1688  end
1689 
1690  // Combine pattern match "subterms" for DQ-IDELAY stage
1691  always @(posedge clk) begin
1692  idel_pat0_match_rise0_and_r <= #TCQ &idel_pat0_match_rise0_r;
1693  idel_pat0_match_fall0_and_r <= #TCQ &idel_pat0_match_fall0_r;
1694  idel_pat0_match_rise1_and_r <= #TCQ &idel_pat0_match_rise1_r;
1695  idel_pat0_match_fall1_and_r <= #TCQ &idel_pat0_match_fall1_r;
1696  idel_pat0_match_rise2_and_r <= #TCQ &idel_pat0_match_rise2_r;
1697  idel_pat0_match_fall2_and_r <= #TCQ &idel_pat0_match_fall2_r;
1698  idel_pat0_match_rise3_and_r <= #TCQ &idel_pat0_match_rise3_r;
1699  idel_pat0_match_fall3_and_r <= #TCQ &idel_pat0_match_fall3_r;
1700  idel_pat0_data_match_r <= #TCQ (idel_pat0_match_rise0_and_r &&
1701  idel_pat0_match_fall0_and_r &&
1702  idel_pat0_match_rise1_and_r &&
1703  idel_pat0_match_fall1_and_r &&
1704  idel_pat0_match_rise2_and_r &&
1705  idel_pat0_match_fall2_and_r &&
1706  idel_pat0_match_rise3_and_r &&
1707  idel_pat0_match_fall3_and_r);
1708  end
1709 
1710  always @(posedge clk) begin
1711  idel_pat1_match_rise0_and_r <= #TCQ &idel_pat1_match_rise0_r;
1712  idel_pat1_match_fall0_and_r <= #TCQ &idel_pat1_match_fall0_r;
1713  idel_pat1_match_rise1_and_r <= #TCQ &idel_pat1_match_rise1_r;
1714  idel_pat1_match_fall1_and_r <= #TCQ &idel_pat1_match_fall1_r;
1715  idel_pat1_match_rise2_and_r <= #TCQ &idel_pat1_match_rise2_r;
1716  idel_pat1_match_fall2_and_r <= #TCQ &idel_pat1_match_fall2_r;
1717  idel_pat1_match_rise3_and_r <= #TCQ &idel_pat1_match_rise3_r;
1718  idel_pat1_match_fall3_and_r <= #TCQ &idel_pat1_match_fall3_r;
1719  idel_pat1_data_match_r <= #TCQ (idel_pat1_match_rise0_and_r &&
1720  idel_pat1_match_fall0_and_r &&
1721  idel_pat1_match_rise1_and_r &&
1722  idel_pat1_match_fall1_and_r &&
1723  idel_pat1_match_rise2_and_r &&
1724  idel_pat1_match_fall2_and_r &&
1725  idel_pat1_match_rise3_and_r &&
1726  idel_pat1_match_fall3_and_r);
1727  end
1728 
1729  always @(idel_pat0_data_match_r or idel_pat1_data_match_r)
1730  idel_pat_data_match <= #TCQ idel_pat0_data_match_r |
1731  idel_pat1_data_match_r;
1732 
1733  always @(posedge clk)
1734  idel_pat_data_match_r <= #TCQ idel_pat_data_match;
1735 
1736  // Combine pattern match "subterms" for DQS-PHASER_IN stage
1737  always @(posedge clk) begin
1738  pat0_match_rise0_and_r <= #TCQ &pat0_match_rise0_r;
1739  pat0_match_fall0_and_r <= #TCQ &pat0_match_fall0_r;
1740  pat0_match_rise1_and_r <= #TCQ &pat0_match_rise1_r;
1741  pat0_match_fall1_and_r <= #TCQ &pat0_match_fall1_r;
1742  pat0_match_rise2_and_r <= #TCQ &pat0_match_rise2_r;
1743  pat0_match_fall2_and_r <= #TCQ &pat0_match_fall2_r;
1744  pat0_match_rise3_and_r <= #TCQ &pat0_match_rise3_r;
1745  pat0_match_fall3_and_r <= #TCQ &pat0_match_fall3_r;
1746  pat0_data_match_r <= #TCQ (pat0_match_rise0_and_r &&
1747  pat0_match_fall0_and_r &&
1748  pat0_match_rise1_and_r &&
1749  pat0_match_fall1_and_r &&
1750  pat0_match_rise2_and_r &&
1751  pat0_match_fall2_and_r &&
1752  pat0_match_rise3_and_r &&
1753  pat0_match_fall3_and_r);
1754  end
1755 
1756  always @(posedge clk) begin
1757  pat1_match_rise0_and_r <= #TCQ &pat1_match_rise0_r;
1758  pat1_match_fall0_and_r <= #TCQ &pat1_match_fall0_r;
1759  pat1_match_rise1_and_r <= #TCQ &pat1_match_rise1_r;
1760  pat1_match_fall1_and_r <= #TCQ &pat1_match_fall1_r;
1761  pat1_match_rise2_and_r <= #TCQ &pat1_match_rise2_r;
1762  pat1_match_fall2_and_r <= #TCQ &pat1_match_fall2_r;
1763  pat1_match_rise3_and_r <= #TCQ &pat1_match_rise3_r;
1764  pat1_match_fall3_and_r <= #TCQ &pat1_match_fall3_r;
1765  pat1_data_match_r <= #TCQ (pat1_match_rise0_and_r &&
1766  pat1_match_fall0_and_r &&
1767  pat1_match_rise1_and_r &&
1768  pat1_match_fall1_and_r &&
1769  pat1_match_rise2_and_r &&
1770  pat1_match_fall2_and_r &&
1771  pat1_match_rise3_and_r &&
1772  pat1_match_fall3_and_r);
1773  end
1774 
1775  assign pat_data_match_r = pat0_data_match_r | pat1_data_match_r;
1776 
1777  end else if (nCK_PER_CLK == 2) begin: gen_pat_match_div2
1778  for (pt_i = 0; pt_i < DRAM_WIDTH; pt_i = pt_i + 1) begin: gen_pat_match
1779 
1780  // DQ IDELAY pattern detection
1781  always @(posedge clk) begin
1782  if (sr_rise0_r[pt_i] == idel_pat0_rise0[pt_i%4])
1783  idel_pat0_match_rise0_r[pt_i] <= #TCQ 1'b1;
1784  else
1785  idel_pat0_match_rise0_r[pt_i] <= #TCQ 1'b0;
1786 
1787  if (sr_fall0_r[pt_i] == idel_pat0_fall0[pt_i%4])
1788  idel_pat0_match_fall0_r[pt_i] <= #TCQ 1'b1;
1789  else
1790  idel_pat0_match_fall0_r[pt_i] <= #TCQ 1'b0;
1791 
1792  if (sr_rise1_r[pt_i] == idel_pat0_rise1[pt_i%4])
1793  idel_pat0_match_rise1_r[pt_i] <= #TCQ 1'b1;
1794  else
1795  idel_pat0_match_rise1_r[pt_i] <= #TCQ 1'b0;
1796 
1797  if (sr_fall1_r[pt_i] == idel_pat0_fall1[pt_i%4])
1798  idel_pat0_match_fall1_r[pt_i] <= #TCQ 1'b1;
1799  else
1800  idel_pat0_match_fall1_r[pt_i] <= #TCQ 1'b0;
1801  end
1802 
1803  always @(posedge clk) begin
1804  if (sr_rise0_r[pt_i] == idel_pat1_rise0[pt_i%4])
1805  idel_pat1_match_rise0_r[pt_i] <= #TCQ 1'b1;
1806  else
1807  idel_pat1_match_rise0_r[pt_i] <= #TCQ 1'b0;
1808 
1809  if (sr_fall0_r[pt_i] == idel_pat1_fall0[pt_i%4])
1810  idel_pat1_match_fall0_r[pt_i] <= #TCQ 1'b1;
1811  else
1812  idel_pat1_match_fall0_r[pt_i] <= #TCQ 1'b0;
1813 
1814  if (sr_rise1_r[pt_i] == idel_pat1_rise1[pt_i%4])
1815  idel_pat1_match_rise1_r[pt_i] <= #TCQ 1'b1;
1816  else
1817  idel_pat1_match_rise1_r[pt_i] <= #TCQ 1'b0;
1818 
1819  if (sr_fall1_r[pt_i] == idel_pat1_fall1[pt_i%4])
1820  idel_pat1_match_fall1_r[pt_i] <= #TCQ 1'b1;
1821  else
1822  idel_pat1_match_fall1_r[pt_i] <= #TCQ 1'b0;
1823  end
1824 
1825  // DQS DVW pattern detection
1826  always @(posedge clk) begin
1827  if (sr_rise0_r[pt_i] == pat0_rise0[pt_i%4])
1828  pat0_match_rise0_r[pt_i] <= #TCQ 1'b1;
1829  else
1830  pat0_match_rise0_r[pt_i] <= #TCQ 1'b0;
1831 
1832  if (sr_fall0_r[pt_i] == pat0_fall0[pt_i%4])
1833  pat0_match_fall0_r[pt_i] <= #TCQ 1'b1;
1834  else
1835  pat0_match_fall0_r[pt_i] <= #TCQ 1'b0;
1836 
1837  if (sr_rise1_r[pt_i] == pat0_rise1[pt_i%4])
1838  pat0_match_rise1_r[pt_i] <= #TCQ 1'b1;
1839  else
1840  pat0_match_rise1_r[pt_i] <= #TCQ 1'b0;
1841 
1842  if (sr_fall1_r[pt_i] == pat0_fall1[pt_i%4])
1843  pat0_match_fall1_r[pt_i] <= #TCQ 1'b1;
1844  else
1845  pat0_match_fall1_r[pt_i] <= #TCQ 1'b0;
1846  end
1847 
1848  always @(posedge clk) begin
1849  if (sr_rise0_r[pt_i] == pat1_rise0[pt_i%4])
1850  pat1_match_rise0_r[pt_i] <= #TCQ 1'b1;
1851  else
1852  pat1_match_rise0_r[pt_i] <= #TCQ 1'b0;
1853 
1854  if (sr_fall0_r[pt_i] == pat1_fall0[pt_i%4])
1855  pat1_match_fall0_r[pt_i] <= #TCQ 1'b1;
1856  else
1857  pat1_match_fall0_r[pt_i] <= #TCQ 1'b0;
1858 
1859  if (sr_rise1_r[pt_i] == pat1_rise1[pt_i%4])
1860  pat1_match_rise1_r[pt_i] <= #TCQ 1'b1;
1861  else
1862  pat1_match_rise1_r[pt_i] <= #TCQ 1'b0;
1863 
1864  if (sr_fall1_r[pt_i] == pat1_fall1[pt_i%4])
1865  pat1_match_fall1_r[pt_i] <= #TCQ 1'b1;
1866  else
1867  pat1_match_fall1_r[pt_i] <= #TCQ 1'b0;
1868  end
1869 
1870  end
1871 
1872  // Combine pattern match "subterms" for DQ-IDELAY stage
1873  always @(posedge clk) begin
1874  idel_pat0_match_rise0_and_r <= #TCQ &idel_pat0_match_rise0_r;
1875  idel_pat0_match_fall0_and_r <= #TCQ &idel_pat0_match_fall0_r;
1876  idel_pat0_match_rise1_and_r <= #TCQ &idel_pat0_match_rise1_r;
1877  idel_pat0_match_fall1_and_r <= #TCQ &idel_pat0_match_fall1_r;
1878  idel_pat0_data_match_r <= #TCQ (idel_pat0_match_rise0_and_r &&
1879  idel_pat0_match_fall0_and_r &&
1880  idel_pat0_match_rise1_and_r &&
1881  idel_pat0_match_fall1_and_r);
1882  end
1883 
1884  always @(posedge clk) begin
1885  idel_pat1_match_rise0_and_r <= #TCQ &idel_pat1_match_rise0_r;
1886  idel_pat1_match_fall0_and_r <= #TCQ &idel_pat1_match_fall0_r;
1887  idel_pat1_match_rise1_and_r <= #TCQ &idel_pat1_match_rise1_r;
1888  idel_pat1_match_fall1_and_r <= #TCQ &idel_pat1_match_fall1_r;
1889  idel_pat1_data_match_r <= #TCQ (idel_pat1_match_rise0_and_r &&
1890  idel_pat1_match_fall0_and_r &&
1891  idel_pat1_match_rise1_and_r &&
1892  idel_pat1_match_fall1_and_r);
1893  end
1894 
1895  always @(posedge clk) begin
1896  if (sr_valid_r2)
1897  idel_pat_data_match <= #TCQ idel_pat0_data_match_r |
1898  idel_pat1_data_match_r;
1899  end
1900 
1901  //assign idel_pat_data_match = idel_pat0_data_match_r |
1902  // idel_pat1_data_match_r;
1903 
1904  always @(posedge clk)
1905  idel_pat_data_match_r <= #TCQ idel_pat_data_match;
1906 
1907  // Combine pattern match "subterms" for DQS-PHASER_IN stage
1908  always @(posedge clk) begin
1909  pat0_match_rise0_and_r <= #TCQ &pat0_match_rise0_r;
1910  pat0_match_fall0_and_r <= #TCQ &pat0_match_fall0_r;
1911  pat0_match_rise1_and_r <= #TCQ &pat0_match_rise1_r;
1912  pat0_match_fall1_and_r <= #TCQ &pat0_match_fall1_r;
1913  pat0_data_match_r <= #TCQ (pat0_match_rise0_and_r &&
1914  pat0_match_fall0_and_r &&
1915  pat0_match_rise1_and_r &&
1916  pat0_match_fall1_and_r);
1917  end
1918 
1919  always @(posedge clk) begin
1920  pat1_match_rise0_and_r <= #TCQ &pat1_match_rise0_r;
1921  pat1_match_fall0_and_r <= #TCQ &pat1_match_fall0_r;
1922  pat1_match_rise1_and_r <= #TCQ &pat1_match_rise1_r;
1923  pat1_match_fall1_and_r <= #TCQ &pat1_match_fall1_r;
1924  pat1_data_match_r <= #TCQ (pat1_match_rise0_and_r &&
1925  pat1_match_fall0_and_r &&
1926  pat1_match_rise1_and_r &&
1927  pat1_match_fall1_and_r);
1928  end
1929 
1930  assign pat_data_match_r = pat0_data_match_r | pat1_data_match_r;
1931 
1932  end
1933 
1934  endgenerate
1935 
1936 
1937  always @(posedge clk) begin
1938  rdlvl_stg1_start_r <= #TCQ rdlvl_stg1_start;
1939  mpr_rdlvl_done_r1 <= #TCQ mpr_rdlvl_done_r;
1940  mpr_rdlvl_done_r2 <= #TCQ mpr_rdlvl_done_r1;
1941  mpr_rdlvl_start_r <= #TCQ mpr_rdlvl_start;
1942  end
1943 
1944  //***************************************************************************
1945  // First stage calibration: Capture clock
1946  //***************************************************************************
1947 
1948  //*****************************************************************
1949  // Keep track of how many samples have been written to shift registers
1950  // Every time RD_SHIFT_LEN samples have been written, then we have a
1951  // full read training pattern loaded into the sr_* registers. Then assert
1952  // sr_valid_r to indicate that: (1) comparison between the sr_* and
1953  // old_sr_* and prev_sr_* registers can take place, (2) transfer of
1954  // the contents of sr_* to old_sr_* and prev_sr_* registers can also
1955  // take place
1956  //*****************************************************************
1957 
1958  always @(posedge clk)
1959  if (rst || (mpr_rdlvl_done_r && ~rdlvl_stg1_start)) begin
1960  cnt_shift_r <= #TCQ 'b1;
1961  sr_valid_r <= #TCQ 1'b0;
1962  mpr_valid_r <= #TCQ 1'b0;
1963  end else begin
1964  if (mux_rd_valid_r && mpr_rdlvl_start && ~mpr_rdlvl_done_r) begin
1965  if (cnt_shift_r == 'b0)
1966  mpr_valid_r <= #TCQ 1'b1;
1967  else begin
1968  mpr_valid_r <= #TCQ 1'b0;
1969  cnt_shift_r <= #TCQ cnt_shift_r + 1;
1970  end
1971  end else
1972  mpr_valid_r <= #TCQ 1'b0;
1973 
1974  if (mux_rd_valid_r && rdlvl_stg1_start) begin
1975  if (cnt_shift_r == RD_SHIFT_LEN-1) begin
1976  sr_valid_r <= #TCQ 1'b1;
1977  cnt_shift_r <= #TCQ 'b0;
1978  end else begin
1979  sr_valid_r <= #TCQ 1'b0;
1980  cnt_shift_r <= #TCQ cnt_shift_r + 1;
1981  end
1982  end else
1983  // When the current mux_rd_* contents are not valid, then
1984  // retain the current value of cnt_shift_r, and make sure
1985  // that sr_valid_r = 0 to prevent any downstream loads or
1986  // comparisons
1987  sr_valid_r <= #TCQ 1'b0;
1988  end
1989 
1990  //*****************************************************************
1991  // Logic to determine when either edge of the data eye encountered
1992  // Pre- and post-IDELAY update data pattern is compared, if they
1993  // differ, than an edge has been encountered. Currently no attempt
1994  // made to determine if the data pattern itself is "correct", only
1995  // whether it changes after incrementing the IDELAY (possible
1996  // future enhancement)
1997  //*****************************************************************
1998 
1999  // One-way control for ensuring that state machine request to store
2000  // current read data into OLD SR shift register only occurs on a
2001  // valid clock cycle. The FSM provides a one-cycle request pulse.
2002  // It is the responsibility of the FSM to wait the worst-case time
2003  // before relying on any downstream results of this load.
2004  always @(posedge clk)
2005  if (rst)
2006  store_sr_r <= #TCQ 1'b0;
2007  else begin
2008  if (store_sr_req_r)
2009  store_sr_r <= #TCQ 1'b1;
2010  else if ((sr_valid_r || mpr_valid_r) && store_sr_r)
2011  store_sr_r <= #TCQ 1'b0;
2012  end
2013 
2014  // Transfer current data to old data, prior to incrementing delay
2015  // Also store data from current sampling window - so that we can detect
2016  // if the current delay tap yields data that is "jittery"
2017  generate
2018  if (nCK_PER_CLK == 4) begin: gen_old_sr_div4
2019  for (z = 0; z < DRAM_WIDTH; z = z + 1) begin: gen_old_sr
2020  always @(posedge clk) begin
2021  if (sr_valid_r || mpr_valid_r) begin
2022  // Load last sample (i.e. from current sampling interval)
2023  prev_sr_rise0_r[z] <= #TCQ sr_rise0_r[z];
2024  prev_sr_fall0_r[z] <= #TCQ sr_fall0_r[z];
2025  prev_sr_rise1_r[z] <= #TCQ sr_rise1_r[z];
2026  prev_sr_fall1_r[z] <= #TCQ sr_fall1_r[z];
2027  prev_sr_rise2_r[z] <= #TCQ sr_rise2_r[z];
2028  prev_sr_fall2_r[z] <= #TCQ sr_fall2_r[z];
2029  prev_sr_rise3_r[z] <= #TCQ sr_rise3_r[z];
2030  prev_sr_fall3_r[z] <= #TCQ sr_fall3_r[z];
2031  end
2032  if ((sr_valid_r || mpr_valid_r) && store_sr_r) begin
2033  old_sr_rise0_r[z] <= #TCQ sr_rise0_r[z];
2034  old_sr_fall0_r[z] <= #TCQ sr_fall0_r[z];
2035  old_sr_rise1_r[z] <= #TCQ sr_rise1_r[z];
2036  old_sr_fall1_r[z] <= #TCQ sr_fall1_r[z];
2037  old_sr_rise2_r[z] <= #TCQ sr_rise2_r[z];
2038  old_sr_fall2_r[z] <= #TCQ sr_fall2_r[z];
2039  old_sr_rise3_r[z] <= #TCQ sr_rise3_r[z];
2040  old_sr_fall3_r[z] <= #TCQ sr_fall3_r[z];
2041  end
2042  end
2043  end
2044  end else if (nCK_PER_CLK == 2) begin: gen_old_sr_div2
2045  for (z = 0; z < DRAM_WIDTH; z = z + 1) begin: gen_old_sr
2046  always @(posedge clk) begin
2047  if (sr_valid_r || mpr_valid_r) begin
2048  prev_sr_rise0_r[z] <= #TCQ sr_rise0_r[z];
2049  prev_sr_fall0_r[z] <= #TCQ sr_fall0_r[z];
2050  prev_sr_rise1_r[z] <= #TCQ sr_rise1_r[z];
2051  prev_sr_fall1_r[z] <= #TCQ sr_fall1_r[z];
2052  end
2053  if ((sr_valid_r || mpr_valid_r) && store_sr_r) begin
2054  old_sr_rise0_r[z] <= #TCQ sr_rise0_r[z];
2055  old_sr_fall0_r[z] <= #TCQ sr_fall0_r[z];
2056  old_sr_rise1_r[z] <= #TCQ sr_rise1_r[z];
2057  old_sr_fall1_r[z] <= #TCQ sr_fall1_r[z];
2058  end
2059  end
2060  end
2061  end
2062  endgenerate
2063 
2064  //*******************************************************
2065  // Match determination occurs over 3 cycles - pipelined for better timing
2066  //*******************************************************
2067 
2068  // Match valid with # of cycles of pipelining in match determination
2069  always @(posedge clk) begin
2070  sr_valid_r1 <= #TCQ sr_valid_r;
2071  sr_valid_r2 <= #TCQ sr_valid_r1;
2072  mpr_valid_r1 <= #TCQ mpr_valid_r;
2073  mpr_valid_r2 <= #TCQ mpr_valid_r1;
2074  end
2075 
2076  generate
2077  if (nCK_PER_CLK == 4) begin: gen_sr_match_div4
2078  for (z = 0; z < DRAM_WIDTH; z = z + 1) begin: gen_sr_match
2079  always @(posedge clk) begin
2080  // CYCLE1: Compare all bits in DQS grp, generate separate term for
2081  // each bit over four bit times. For example, if there are 8-bits
2082  // per DQS group, 32 terms are generated on cycle 1
2083  // NOTE: Structure HDL such that X on data bus will result in a
2084  // mismatch. This is required for memory models that can drive the
2085  // bus with X's to model uncertainty regions (e.g. Denali)
2086  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise0_r[z] == old_sr_rise0_r[z]))
2087  old_sr_match_rise0_r[z] <= #TCQ 1'b1;
2088  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2089  old_sr_match_rise0_r[z] <= #TCQ old_sr_match_rise0_r[z];
2090  else
2091  old_sr_match_rise0_r[z] <= #TCQ 1'b0;
2092 
2093  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall0_r[z] == old_sr_fall0_r[z]))
2094  old_sr_match_fall0_r[z] <= #TCQ 1'b1;
2095  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2096  old_sr_match_fall0_r[z] <= #TCQ old_sr_match_fall0_r[z];
2097  else
2098  old_sr_match_fall0_r[z] <= #TCQ 1'b0;
2099 
2100  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise1_r[z] == old_sr_rise1_r[z]))
2101  old_sr_match_rise1_r[z] <= #TCQ 1'b1;
2102  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2103  old_sr_match_rise1_r[z] <= #TCQ old_sr_match_rise1_r[z];
2104  else
2105  old_sr_match_rise1_r[z] <= #TCQ 1'b0;
2106 
2107  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall1_r[z] == old_sr_fall1_r[z]))
2108  old_sr_match_fall1_r[z] <= #TCQ 1'b1;
2109  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2110  old_sr_match_fall1_r[z] <= #TCQ old_sr_match_fall1_r[z];
2111  else
2112  old_sr_match_fall1_r[z] <= #TCQ 1'b0;
2113 
2114  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise2_r[z] == old_sr_rise2_r[z]))
2115  old_sr_match_rise2_r[z] <= #TCQ 1'b1;
2116  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2117  old_sr_match_rise2_r[z] <= #TCQ old_sr_match_rise2_r[z];
2118  else
2119  old_sr_match_rise2_r[z] <= #TCQ 1'b0;
2120 
2121  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall2_r[z] == old_sr_fall2_r[z]))
2122  old_sr_match_fall2_r[z] <= #TCQ 1'b1;
2123  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2124  old_sr_match_fall2_r[z] <= #TCQ old_sr_match_fall2_r[z];
2125  else
2126  old_sr_match_fall2_r[z] <= #TCQ 1'b0;
2127 
2128  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise3_r[z] == old_sr_rise3_r[z]))
2129  old_sr_match_rise3_r[z] <= #TCQ 1'b1;
2130  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2131  old_sr_match_rise3_r[z] <= #TCQ old_sr_match_rise3_r[z];
2132  else
2133  old_sr_match_rise3_r[z] <= #TCQ 1'b0;
2134 
2135  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall3_r[z] == old_sr_fall3_r[z]))
2136  old_sr_match_fall3_r[z] <= #TCQ 1'b1;
2137  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2138  old_sr_match_fall3_r[z] <= #TCQ old_sr_match_fall3_r[z];
2139  else
2140  old_sr_match_fall3_r[z] <= #TCQ 1'b0;
2141 
2142  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise0_r[z] == prev_sr_rise0_r[z]))
2143  prev_sr_match_rise0_r[z] <= #TCQ 1'b1;
2144  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2145  prev_sr_match_rise0_r[z] <= #TCQ prev_sr_match_rise0_r[z];
2146  else
2147  prev_sr_match_rise0_r[z] <= #TCQ 1'b0;
2148 
2149  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall0_r[z] == prev_sr_fall0_r[z]))
2150  prev_sr_match_fall0_r[z] <= #TCQ 1'b1;
2151  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2152  prev_sr_match_fall0_r[z] <= #TCQ prev_sr_match_fall0_r[z];
2153  else
2154  prev_sr_match_fall0_r[z] <= #TCQ 1'b0;
2155 
2156  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise1_r[z] == prev_sr_rise1_r[z]))
2157  prev_sr_match_rise1_r[z] <= #TCQ 1'b1;
2158  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2159  prev_sr_match_rise1_r[z] <= #TCQ prev_sr_match_rise1_r[z];
2160  else
2161  prev_sr_match_rise1_r[z] <= #TCQ 1'b0;
2162 
2163  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall1_r[z] == prev_sr_fall1_r[z]))
2164  prev_sr_match_fall1_r[z] <= #TCQ 1'b1;
2165  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2166  prev_sr_match_fall1_r[z] <= #TCQ prev_sr_match_fall1_r[z];
2167  else
2168  prev_sr_match_fall1_r[z] <= #TCQ 1'b0;
2169 
2170  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise2_r[z] == prev_sr_rise2_r[z]))
2171  prev_sr_match_rise2_r[z] <= #TCQ 1'b1;
2172  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2173  prev_sr_match_rise2_r[z] <= #TCQ prev_sr_match_rise2_r[z];
2174  else
2175  prev_sr_match_rise2_r[z] <= #TCQ 1'b0;
2176 
2177  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall2_r[z] == prev_sr_fall2_r[z]))
2178  prev_sr_match_fall2_r[z] <= #TCQ 1'b1;
2179  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2180  prev_sr_match_fall2_r[z] <= #TCQ prev_sr_match_fall2_r[z];
2181  else
2182  prev_sr_match_fall2_r[z] <= #TCQ 1'b0;
2183 
2184  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise3_r[z] == prev_sr_rise3_r[z]))
2185  prev_sr_match_rise3_r[z] <= #TCQ 1'b1;
2186  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2187  prev_sr_match_rise3_r[z] <= #TCQ prev_sr_match_rise3_r[z];
2188  else
2189  prev_sr_match_rise3_r[z] <= #TCQ 1'b0;
2190 
2191  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall3_r[z] == prev_sr_fall3_r[z]))
2192  prev_sr_match_fall3_r[z] <= #TCQ 1'b1;
2193  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2194  prev_sr_match_fall3_r[z] <= #TCQ prev_sr_match_fall3_r[z];
2195  else
2196  prev_sr_match_fall3_r[z] <= #TCQ 1'b0;
2197 
2198  // CYCLE2: Combine all the comparisons for every 8 words (rise0,
2199  // fall0,rise1, fall1) in the calibration sequence. Now we're down
2200  // to DRAM_WIDTH terms
2201  old_sr_match_cyc2_r[z] <= #TCQ
2202  old_sr_match_rise0_r[z] &
2203  old_sr_match_fall0_r[z] &
2204  old_sr_match_rise1_r[z] &
2205  old_sr_match_fall1_r[z] &
2206  old_sr_match_rise2_r[z] &
2207  old_sr_match_fall2_r[z] &
2208  old_sr_match_rise3_r[z] &
2209  old_sr_match_fall3_r[z];
2210  prev_sr_match_cyc2_r[z] <= #TCQ
2211  prev_sr_match_rise0_r[z] &
2212  prev_sr_match_fall0_r[z] &
2213  prev_sr_match_rise1_r[z] &
2214  prev_sr_match_fall1_r[z] &
2215  prev_sr_match_rise2_r[z] &
2216  prev_sr_match_fall2_r[z] &
2217  prev_sr_match_rise3_r[z] &
2218  prev_sr_match_fall3_r[z];
2219 
2220  // CYCLE3: Invert value (i.e. assert when DIFFERENCE in value seen),
2221  // and qualify with pipelined valid signal) - probably don't need
2222  // a cycle just do do this....
2223  if (sr_valid_r2 || mpr_valid_r2) begin
2224  old_sr_diff_r[z] <= #TCQ ~old_sr_match_cyc2_r[z];
2225  prev_sr_diff_r[z] <= #TCQ ~prev_sr_match_cyc2_r[z];
2226  end else begin
2227  old_sr_diff_r[z] <= #TCQ 'b0;
2228  prev_sr_diff_r[z] <= #TCQ 'b0;
2229  end
2230  end
2231  end
2232  end if (nCK_PER_CLK == 2) begin: gen_sr_match_div2
2233  for (z = 0; z < DRAM_WIDTH; z = z + 1) begin: gen_sr_match
2234  always @(posedge clk) begin
2235  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise0_r[z] == old_sr_rise0_r[z]))
2236  old_sr_match_rise0_r[z] <= #TCQ 1'b1;
2237  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2238  old_sr_match_rise0_r[z] <= #TCQ old_sr_match_rise0_r[z];
2239  else
2240  old_sr_match_rise0_r[z] <= #TCQ 1'b0;
2241 
2242  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall0_r[z] == old_sr_fall0_r[z]))
2243  old_sr_match_fall0_r[z] <= #TCQ 1'b1;
2244  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2245  old_sr_match_fall0_r[z] <= #TCQ old_sr_match_fall0_r[z];
2246  else
2247  old_sr_match_fall0_r[z] <= #TCQ 1'b0;
2248 
2249  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise1_r[z] == old_sr_rise1_r[z]))
2250  old_sr_match_rise1_r[z] <= #TCQ 1'b1;
2251  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2252  old_sr_match_rise1_r[z] <= #TCQ old_sr_match_rise1_r[z];
2253  else
2254  old_sr_match_rise1_r[z] <= #TCQ 1'b0;
2255 
2256  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall1_r[z] == old_sr_fall1_r[z]))
2257  old_sr_match_fall1_r[z] <= #TCQ 1'b1;
2258  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2259  old_sr_match_fall1_r[z] <= #TCQ old_sr_match_fall1_r[z];
2260  else
2261  old_sr_match_fall1_r[z] <= #TCQ 1'b0;
2262 
2263  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise0_r[z] == prev_sr_rise0_r[z]))
2264  prev_sr_match_rise0_r[z] <= #TCQ 1'b1;
2265  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2266  prev_sr_match_rise0_r[z] <= #TCQ prev_sr_match_rise0_r[z];
2267  else
2268  prev_sr_match_rise0_r[z] <= #TCQ 1'b0;
2269 
2270  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall0_r[z] == prev_sr_fall0_r[z]))
2271  prev_sr_match_fall0_r[z] <= #TCQ 1'b1;
2272  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2273  prev_sr_match_fall0_r[z] <= #TCQ prev_sr_match_fall0_r[z];
2274  else
2275  prev_sr_match_fall0_r[z] <= #TCQ 1'b0;
2276 
2277  if ((pat_data_match_r || mpr_valid_r1) && (sr_rise1_r[z] == prev_sr_rise1_r[z]))
2278  prev_sr_match_rise1_r[z] <= #TCQ 1'b1;
2279  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2280  prev_sr_match_rise1_r[z] <= #TCQ prev_sr_match_rise1_r[z];
2281  else
2282  prev_sr_match_rise1_r[z] <= #TCQ 1'b0;
2283 
2284  if ((pat_data_match_r || mpr_valid_r1) && (sr_fall1_r[z] == prev_sr_fall1_r[z]))
2285  prev_sr_match_fall1_r[z] <= #TCQ 1'b1;
2286  else if (~mpr_valid_r1 && mpr_rdlvl_start && ~mpr_rdlvl_done_r)
2287  prev_sr_match_fall1_r[z] <= #TCQ prev_sr_match_fall1_r[z];
2288  else
2289  prev_sr_match_fall1_r[z] <= #TCQ 1'b0;
2290 
2291  old_sr_match_cyc2_r[z] <= #TCQ
2292  old_sr_match_rise0_r[z] &
2293  old_sr_match_fall0_r[z] &
2294  old_sr_match_rise1_r[z] &
2295  old_sr_match_fall1_r[z];
2296  prev_sr_match_cyc2_r[z] <= #TCQ
2297  prev_sr_match_rise0_r[z] &
2298  prev_sr_match_fall0_r[z] &
2299  prev_sr_match_rise1_r[z] &
2300  prev_sr_match_fall1_r[z];
2301 
2302  // CYCLE3: Invert value (i.e. assert when DIFFERENCE in value seen),
2303  // and qualify with pipelined valid signal) - probably don't need
2304  // a cycle just do do this....
2305  if (sr_valid_r2 || mpr_valid_r2) begin
2306  old_sr_diff_r[z] <= #TCQ ~old_sr_match_cyc2_r[z];
2307  prev_sr_diff_r[z] <= #TCQ ~prev_sr_match_cyc2_r[z];
2308  end else begin
2309  old_sr_diff_r[z] <= #TCQ 'b0;
2310  prev_sr_diff_r[z] <= #TCQ 'b0;
2311  end
2312  end
2313  end
2314  end
2315  endgenerate
2316 
2317  //***************************************************************************
2318  // First stage calibration: DQS Capture
2319  //***************************************************************************
2320 
2321 
2322  //*******************************************************
2323  // Counters for tracking # of samples compared
2324  // For each comparision point (i.e. to determine if an edge has
2325  // occurred after each IODELAY increment when read leveling),
2326  // multiple samples are compared in order to average out the effects
2327  // of jitter. If any one of these samples is different than the "old"
2328  // sample corresponding to the previous IODELAY value, then an edge
2329  // is declared to be detected.
2330  //*******************************************************
2331 
2332  // Two cascaded counters are used to keep track of # of samples compared,
2333  // in order to make it easier to meeting timing on these paths. Once
2334  // optimal sampling interval is determined, it may be possible to remove
2335  // the second counter
2336  always @(posedge clk)
2337  samp_edge_cnt0_en_r <= #TCQ
2338  (cal1_state_r == CAL1_PAT_DETECT) ||
2339  (cal1_state_r == CAL1_DETECT_EDGE) ||
2340  (cal1_state_r == CAL1_PB_DETECT_EDGE) ||
2341  (cal1_state_r == CAL1_PB_DETECT_EDGE_DQ);
2342 
2343  // First counter counts # of samples compared
2344  always @(posedge clk)
2345  if (rst)
2346  samp_edge_cnt0_r <= #TCQ 'b0;
2347  else begin
2348  if (!samp_edge_cnt0_en_r)
2349  // Reset sample counter when not in any of the "sampling" states
2350  samp_edge_cnt0_r <= #TCQ 'b0;
2351  else if (sr_valid_r2 || mpr_valid_r2)
2352  // Otherwise, count # of samples compared
2353  samp_edge_cnt0_r <= #TCQ samp_edge_cnt0_r + 1;
2354  end
2355 
2356  // Counter #2 enable generation
2357  always @(posedge clk)
2358  if (rst)
2359  samp_edge_cnt1_en_r <= #TCQ 1'b0;
2360  else begin
2361  // Assert pulse when correct number of samples compared
2362  if ((samp_edge_cnt0_r == DETECT_EDGE_SAMPLE_CNT0) &&
2363  (sr_valid_r2 || mpr_valid_r2))
2364  samp_edge_cnt1_en_r <= #TCQ 1'b1;
2365  else
2366  samp_edge_cnt1_en_r <= #TCQ 1'b0;
2367  end
2368 
2369  // Counter #2
2370  always @(posedge clk)
2371  if (rst)
2372  samp_edge_cnt1_r <= #TCQ 'b0;
2373  else
2374  if (!samp_edge_cnt0_en_r)
2375  samp_edge_cnt1_r <= #TCQ 'b0;
2376  else if (samp_edge_cnt1_en_r)
2377  samp_edge_cnt1_r <= #TCQ samp_edge_cnt1_r + 1;
2378 
2379  always @(posedge clk)
2380  if (rst)
2381  samp_cnt_done_r <= #TCQ 1'b0;
2382  else begin
2383  if (!samp_edge_cnt0_en_r)
2384  samp_cnt_done_r <= #TCQ 'b0;
2385  else if ((SIM_CAL_OPTION == "FAST_CAL") ||
2386  (SIM_CAL_OPTION == "FAST_WIN_DETECT")) begin
2387  if (samp_edge_cnt0_r == SR_VALID_DELAY-1)
2388  // For simulation only, stay in edge detection mode a minimum
2389  // amount of time - just enough for two data compares to finish
2390  samp_cnt_done_r <= #TCQ 1'b1;
2391  end else begin
2392  if (samp_edge_cnt1_r == DETECT_EDGE_SAMPLE_CNT1)
2393  samp_cnt_done_r <= #TCQ 1'b1;
2394  end
2395  end
2396 
2397  //*****************************************************************
2398  // Logic to keep track of (on per-bit basis):
2399  // 1. When a region of stability preceded by a known edge occurs
2400  // 2. If for the current tap, the read data jitters
2401  // 3. If an edge occured between the current and previous tap
2402  // 4. When the current edge detection/sampling interval can end
2403  // Essentially, these are a series of status bits - the stage 1
2404  // calibration FSM monitors these to determine when an edge is
2405  // found. Additional information is provided to help the FSM
2406  // determine if a left or right edge has been found.
2407  //****************************************************************
2408 
2409  assign pb_detect_edge_setup
2410  = (cal1_state_r == CAL1_STORE_FIRST_WAIT) ||
2411  (cal1_state_r == CAL1_PB_STORE_FIRST_WAIT) ||
2412  (cal1_state_r == CAL1_PB_DEC_CPT_LEFT_WAIT);
2413 
2414  assign pb_detect_edge
2415  = (cal1_state_r == CAL1_PAT_DETECT) ||
2416  (cal1_state_r == CAL1_DETECT_EDGE) ||
2417  (cal1_state_r == CAL1_PB_DETECT_EDGE) ||
2418  (cal1_state_r == CAL1_PB_DETECT_EDGE_DQ);
2419 
2420  generate
2421  for (z = 0; z < DRAM_WIDTH; z = z + 1) begin: gen_track_left_edge
2422  always @(posedge clk) begin
2423  if (pb_detect_edge_setup) begin
2424  // Reset eye size, stable eye marker, and jitter marker before
2425  // starting new edge detection iteration
2426  pb_cnt_eye_size_r[z] <= #TCQ 5'd0;
2427  pb_detect_edge_done_r[z] <= #TCQ 1'b0;
2428  pb_found_stable_eye_r[z] <= #TCQ 1'b0;
2429  pb_last_tap_jitter_r[z] <= #TCQ 1'b0;
2430  pb_found_edge_last_r[z] <= #TCQ 1'b0;
2431  pb_found_edge_r[z] <= #TCQ 1'b0;
2432  pb_found_first_edge_r[z] <= #TCQ 1'b0;
2433  end else if (pb_detect_edge) begin
2434  // Save information on which DQ bits are already out of the
2435  // data valid window - those DQ bits will later not have their
2436  // IDELAY tap value incremented
2437  pb_found_edge_last_r[z] <= #TCQ pb_found_edge_r[z];
2438 
2439  if (!pb_detect_edge_done_r[z]) begin
2440  if (samp_cnt_done_r) begin
2441  // If we've reached end of sampling interval, no jitter on
2442  // current tap has been found (although an edge could have
2443  // been found between the current and previous taps), and
2444  // the sampling interval is complete. Increment the stable
2445  // eye counter if no edge found, and always clear the jitter
2446  // flag in preparation for the next tap.
2447  pb_last_tap_jitter_r[z] <= #TCQ 1'b0;
2448  pb_detect_edge_done_r[z] <= #TCQ 1'b1;
2449  if (!pb_found_edge_r[z] && !pb_last_tap_jitter_r[z]) begin
2450  // If the data was completely stable during this tap and
2451  // no edge was found between this and the previous tap
2452  // then increment the stable eye counter "as appropriate"
2453  if (pb_cnt_eye_size_r[z] != MIN_EYE_SIZE-1)
2454  pb_cnt_eye_size_r[z] <= #TCQ pb_cnt_eye_size_r[z] + 1;
2455  else //if (pb_found_first_edge_r[z])
2456  // We've reached minimum stable eye width
2457  pb_found_stable_eye_r[z] <= #TCQ 1'b1;
2458  end else begin
2459  // Otherwise, an edge was found, either because of a
2460  // difference between this and the previous tap's read
2461  // data, and/or because the previous tap's data jittered
2462  // (but not the current tap's data), then just set the
2463  // edge found flag, and enable the stable eye counter
2464  pb_cnt_eye_size_r[z] <= #TCQ 5'd0;
2465  pb_found_stable_eye_r[z] <= #TCQ 1'b0;
2466  pb_found_edge_r[z] <= #TCQ 1'b1;
2467  pb_detect_edge_done_r[z] <= #TCQ 1'b1;
2468  end
2469  end else if (prev_sr_diff_r[z]) begin
2470  // If we find that the current tap read data jitters, then
2471  // set edge and jitter found flags, "enable" the eye size
2472  // counter, and stop sampling interval for this bit
2473  pb_cnt_eye_size_r[z] <= #TCQ 5'd0;
2474  pb_found_stable_eye_r[z] <= #TCQ 1'b0;
2475  pb_last_tap_jitter_r[z] <= #TCQ 1'b1;
2476  pb_found_edge_r[z] <= #TCQ 1'b1;
2477  pb_found_first_edge_r[z] <= #TCQ 1'b1;
2478  pb_detect_edge_done_r[z] <= #TCQ 1'b1;
2479  end else if (old_sr_diff_r[z] || pb_last_tap_jitter_r[z]) begin
2480  // If either an edge was found (i.e. difference between
2481  // current tap and previous tap read data), or the previous
2482  // tap exhibited jitter (which means by definition that the
2483  // current tap cannot match the previous tap because the
2484  // previous tap gave unstable data), then set the edge found
2485  // flag, and "enable" eye size counter. But do not stop
2486  // sampling interval - we still need to check if the current
2487  // tap exhibits jitter
2488  pb_cnt_eye_size_r[z] <= #TCQ 5'd0;
2489  pb_found_stable_eye_r[z] <= #TCQ 1'b0;
2490  pb_found_edge_r[z] <= #TCQ 1'b1;
2491  pb_found_first_edge_r[z] <= #TCQ 1'b1;
2492  end
2493  end
2494  end else begin
2495  // Before every edge detection interval, reset "intra-tap" flags
2496  pb_found_edge_r[z] <= #TCQ 1'b0;
2497  pb_detect_edge_done_r[z] <= #TCQ 1'b0;
2498  end
2499  end
2500  end
2501  endgenerate
2502 
2503  // Combine the above per-bit status flags into combined terms when
2504  // performing deskew on the aggregate data window
2505  always @(posedge clk) begin
2506  detect_edge_done_r <= #TCQ &pb_detect_edge_done_r;
2507  found_edge_r <= #TCQ |pb_found_edge_r;
2508  found_edge_all_r <= #TCQ &pb_found_edge_r;
2509  found_stable_eye_r <= #TCQ &pb_found_stable_eye_r;
2510  end
2511 
2512  // last IODELAY "stable eye" indicator is updated only after
2513  // detect_edge_done_r is asserted - so that when we do find the "right edge"
2514  // of the data valid window, found_edge_r = 1, AND found_stable_eye_r = 1
2515  // when detect_edge_done_r = 1 (otherwise, if found_stable_eye_r updates
2516  // immediately, then it never possible to have found_stable_eye_r = 1
2517  // when we detect an edge - and we'll never know whether we've found
2518  // a "right edge")
2519  always @(posedge clk)
2520  if (pb_detect_edge_setup)
2521  found_stable_eye_last_r <= #TCQ 1'b0;
2522  else if (detect_edge_done_r)
2523  found_stable_eye_last_r <= #TCQ found_stable_eye_r;
2524 
2525  //*****************************************************************
2526  // Keep track of DQ IDELAYE2 taps used
2527  //*****************************************************************
2528 
2529  // Added additional register stage to improve timing
2530  always @(posedge clk)
2531  if (rst)
2532  idelay_tap_cnt_slice_r <= 5'h0;
2533  else
2534  idelay_tap_cnt_slice_r <= idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing];
2535 
2536  always @(posedge clk)
2537  if (rst || (SIM_CAL_OPTION == "SKIP_CAL")) begin //|| new_cnt_cpt_r
2538  for (s = 0; s < RANKS; s = s + 1) begin
2539  for (t = 0; t < DQS_WIDTH; t = t + 1) begin
2540  idelay_tap_cnt_r[s][t] <= #TCQ idelaye2_init_val;
2541  end
2542  end
2543  end else if (SIM_CAL_OPTION == "FAST_CAL") begin
2544  for (u = 0; u < RANKS; u = u + 1) begin
2545  for (w = 0; w < DQS_WIDTH; w = w + 1) begin
2546  if (cal1_dq_idel_ce) begin
2547  if (cal1_dq_idel_inc)
2548  idelay_tap_cnt_r[u][w] <= #TCQ idelay_tap_cnt_r[u][w] + 1;
2549  else
2550  idelay_tap_cnt_r[u][w] <= #TCQ idelay_tap_cnt_r[u][w] - 1;
2551  end
2552  end
2553  end
2554  end else if ((rnk_cnt_r == RANKS-1) && (RANKS == 2) &&
2555  rdlvl_rank_done_r && (cal1_state_r == CAL1_IDLE)) begin
2556  for (f = 0; f < DQS_WIDTH; f = f + 1) begin
2557  idelay_tap_cnt_r[rnk_cnt_r][f] <= #TCQ idelay_tap_cnt_r[(rnk_cnt_r-1)][f];
2558  end
2559  end else if (cal1_dq_idel_ce) begin
2560  if (cal1_dq_idel_inc)
2561  idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing] <= #TCQ idelay_tap_cnt_slice_r + 5'h1;
2562  else
2563  idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing] <= #TCQ idelay_tap_cnt_slice_r - 5'h1;
2564  end else if (idelay_ld)
2565  idelay_tap_cnt_r[0][wrcal_cnt] <= #TCQ 5'b00000;
2566 
2567  always @(posedge clk)
2568  if (rst || new_cnt_cpt_r)
2569  idelay_tap_limit_r <= #TCQ 1'b0;
2570  else if (idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_r] == 'd31)
2571  idelay_tap_limit_r <= #TCQ 1'b1;
2572 
2573  //*****************************************************************
2574  // keep track of edge tap counts found, and current capture clock
2575  // tap count
2576  //*****************************************************************
2577 
2578  always @(posedge clk)
2579  if (rst || new_cnt_cpt_r ||
2580  (mpr_rdlvl_done_r1 && ~mpr_rdlvl_done_r2))
2581  tap_cnt_cpt_r <= #TCQ 'b0;
2582  else if (cal1_dlyce_cpt_r) begin
2583  if (cal1_dlyinc_cpt_r)
2584  tap_cnt_cpt_r <= #TCQ tap_cnt_cpt_r + 1;
2585  else if (tap_cnt_cpt_r != 'd0)
2586  tap_cnt_cpt_r <= #TCQ tap_cnt_cpt_r - 1;
2587  end
2588 
2589  always @(posedge clk)
2590  if (rst || new_cnt_cpt_r ||
2591  (cal1_state_r1 == CAL1_DQ_IDEL_TAP_INC) ||
2592  (mpr_rdlvl_done_r1 && ~mpr_rdlvl_done_r2))
2593  tap_limit_cpt_r <= #TCQ 1'b0;
2594  else if (tap_cnt_cpt_r == 6'd63)
2595  tap_limit_cpt_r <= #TCQ 1'b1;
2596 
2597  always @(posedge clk)
2598  cal1_cnt_cpt_timing_r <= #TCQ cal1_cnt_cpt_r;
2599 
2600  assign cal1_cnt_cpt_timing = {2'b00, cal1_cnt_cpt_r};
2601 
2602  // Storing DQS tap values at the end of each DQS read leveling
2603  always @(posedge clk) begin
2604  if (rst) begin
2605  for (a = 0; a < RANKS; a = a + 1) begin: rst_rdlvl_dqs_tap_count_loop
2606  for (b = 0; b < DQS_WIDTH; b = b + 1)
2607  rdlvl_dqs_tap_cnt_r[a][b] <= #TCQ 'b0;
2608  end
2609  end else if ((SIM_CAL_OPTION == "FAST_CAL") & (cal1_state_r1 == CAL1_NEXT_DQS)) begin
2610  for (p = 0; p < RANKS; p = p +1) begin: rdlvl_dqs_tap_rank_cnt
2611  for(q = 0; q < DQS_WIDTH; q = q +1) begin: rdlvl_dqs_tap_cnt
2612  rdlvl_dqs_tap_cnt_r[p][q] <= #TCQ tap_cnt_cpt_r;
2613  end
2614  end
2615  end else if (SIM_CAL_OPTION == "SKIP_CAL") begin
2616  for (j = 0; j < RANKS; j = j +1) begin: rdlvl_dqs_tap_rnk_cnt
2617  for(i = 0; i < DQS_WIDTH; i = i +1) begin: rdlvl_dqs_cnt
2618  rdlvl_dqs_tap_cnt_r[j][i] <= #TCQ 6'd31;
2619  end
2620  end
2621  end else if (cal1_state_r1 == CAL1_NEXT_DQS) begin
2622  rdlvl_dqs_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing_r] <= #TCQ tap_cnt_cpt_r;
2623  end
2624  end
2625 
2626 
2627  // Counter to track maximum DQ IODELAY tap usage during the per-bit
2628  // deskew portion of stage 1 calibration
2629  always @(posedge clk)
2630  if (rst) begin
2631  idel_tap_cnt_dq_pb_r <= #TCQ 'b0;
2632  idel_tap_limit_dq_pb_r <= #TCQ 1'b0;
2633  end else
2634  if (new_cnt_cpt_r) begin
2635  idel_tap_cnt_dq_pb_r <= #TCQ 'b0;
2636  idel_tap_limit_dq_pb_r <= #TCQ 1'b0;
2637  end else if (|cal1_dlyce_dq_r) begin
2638  if (cal1_dlyinc_dq_r)
2639  idel_tap_cnt_dq_pb_r <= #TCQ idel_tap_cnt_dq_pb_r + 1;
2640  else
2641  idel_tap_cnt_dq_pb_r <= #TCQ idel_tap_cnt_dq_pb_r - 1;
2642 
2643  if (idel_tap_cnt_dq_pb_r == 31)
2644  idel_tap_limit_dq_pb_r <= #TCQ 1'b1;
2645  else
2646  idel_tap_limit_dq_pb_r <= #TCQ 1'b0;
2647  end
2648 
2649 
2650  //*****************************************************************
2651 
2652  always @(posedge clk)
2653  cal1_state_r1 <= #TCQ cal1_state_r;
2654 
2655  always @(posedge clk)
2656  if (rst) begin
2657  cal1_cnt_cpt_r <= #TCQ 'b0;
2658  cal1_dlyce_cpt_r <= #TCQ 1'b0;
2659  cal1_dlyinc_cpt_r <= #TCQ 1'b0;
2660  cal1_dq_idel_ce <= #TCQ 1'b0;
2661  cal1_dq_idel_inc <= #TCQ 1'b0;
2662  cal1_prech_req_r <= #TCQ 1'b0;
2663  cal1_state_r <= #TCQ CAL1_IDLE;
2664  cnt_idel_dec_cpt_r <= #TCQ 6'bxxxxxx;
2665  found_first_edge_r <= #TCQ 1'b0;
2666  found_second_edge_r <= #TCQ 1'b0;
2667  right_edge_taps_r <= #TCQ 6'bxxxxxx;
2668  first_edge_taps_r <= #TCQ 6'bxxxxxx;
2669  new_cnt_cpt_r <= #TCQ 1'b0;
2670  rdlvl_stg1_done <= #TCQ 1'b0;
2671  rdlvl_stg1_err <= #TCQ 1'b0;
2672  second_edge_taps_r <= #TCQ 6'bxxxxxx;
2673  store_sr_req_pulsed_r <= #TCQ 1'b0;
2674  store_sr_req_r <= #TCQ 1'b0;
2675  rnk_cnt_r <= #TCQ 2'b00;
2676  rdlvl_rank_done_r <= #TCQ 1'b0;
2677  idel_dec_cnt <= #TCQ 'd0;
2678  rdlvl_last_byte_done <= #TCQ 1'b0;
2679  idel_pat_detect_valid_r <= #TCQ 1'b0;
2680  mpr_rank_done_r <= #TCQ 1'b0;
2681  mpr_last_byte_done <= #TCQ 1'b0;
2682  if (OCAL_EN == "ON")
2683  mpr_rdlvl_done_r <= #TCQ 1'b0;
2684  else
2685  mpr_rdlvl_done_r <= #TCQ 1'b1;
2686  mpr_dec_cpt_r <= #TCQ 1'b0;
2687  end else begin
2688  // default (inactive) states for all "pulse" outputs
2689  cal1_prech_req_r <= #TCQ 1'b0;
2690  cal1_dlyce_cpt_r <= #TCQ 1'b0;
2691  cal1_dlyinc_cpt_r <= #TCQ 1'b0;
2692  cal1_dq_idel_ce <= #TCQ 1'b0;
2693  cal1_dq_idel_inc <= #TCQ 1'b0;
2694  new_cnt_cpt_r <= #TCQ 1'b0;
2695  store_sr_req_pulsed_r <= #TCQ 1'b0;
2696  store_sr_req_r <= #TCQ 1'b0;
2697 
2698  case (cal1_state_r)
2699 
2700  CAL1_IDLE: begin
2701  rdlvl_rank_done_r <= #TCQ 1'b0;
2702  rdlvl_last_byte_done <= #TCQ 1'b0;
2703  mpr_rank_done_r <= #TCQ 1'b0;
2704  mpr_last_byte_done <= #TCQ 1'b0;
2705  if (mpr_rdlvl_start && ~mpr_rdlvl_start_r) begin
2706  cal1_state_r <= #TCQ CAL1_MPR_NEW_DQS_WAIT;
2707  end else
2708  if (rdlvl_stg1_start && ~rdlvl_stg1_start_r) begin
2709  if (SIM_CAL_OPTION == "SKIP_CAL")
2710  cal1_state_r <= #TCQ CAL1_REGL_LOAD;
2711  else if (SIM_CAL_OPTION == "FAST_CAL")
2712  cal1_state_r <= #TCQ CAL1_NEXT_DQS;
2713  else begin
2714  new_cnt_cpt_r <= #TCQ 1'b1;
2715  cal1_state_r <= #TCQ CAL1_NEW_DQS_WAIT;
2716  end
2717  end
2718  end
2719 
2720  CAL1_MPR_NEW_DQS_WAIT: begin
2721  cal1_prech_req_r <= #TCQ 1'b0;
2722  if (!cal1_wait_r && mpr_valid_r)
2723  cal1_state_r <= #TCQ CAL1_MPR_PAT_DETECT;
2724  end
2725 
2726  // Wait for the new DQS group to change
2727  // also gives time for the read data IN_FIFO to
2728  // output the updated data for the new DQS group
2729  CAL1_NEW_DQS_WAIT: begin
2730  rdlvl_rank_done_r <= #TCQ 1'b0;
2731  rdlvl_last_byte_done <= #TCQ 1'b0;
2732  mpr_rank_done_r <= #TCQ 1'b0;
2733  mpr_last_byte_done <= #TCQ 1'b0;
2734  cal1_prech_req_r <= #TCQ 1'b0;
2735  if (|pi_counter_read_val) begin //VK_REVIEW
2736  mpr_dec_cpt_r <= #TCQ 1'b1;
2737  cal1_state_r <= #TCQ CAL1_IDEL_DEC_CPT;
2738  cnt_idel_dec_cpt_r <= #TCQ pi_counter_read_val;
2739  end else if (!cal1_wait_r) begin
2740  //if (!cal1_wait_r) begin
2741  // Store "previous tap" read data. Technically there is no
2742  // "previous" read data, since we are starting a new DQS
2743  // group, so we'll never find an edge at tap 0 unless the
2744  // data is fluctuating/jittering
2745  store_sr_req_r <= #TCQ 1'b1;
2746  // If per-bit deskew is disabled, then skip the first
2747  // portion of stage 1 calibration
2748  if (PER_BIT_DESKEW == "OFF")
2749  cal1_state_r <= #TCQ CAL1_STORE_FIRST_WAIT;
2750  else if (PER_BIT_DESKEW == "ON")
2751  cal1_state_r <= #TCQ CAL1_PB_STORE_FIRST_WAIT;
2752  end
2753  end
2754  //*****************************************************************
2755  // Per-bit deskew states
2756  //*****************************************************************
2757 
2758  // Wait state following storage of initial read data
2759  CAL1_PB_STORE_FIRST_WAIT:
2760  if (!cal1_wait_r)
2761  cal1_state_r <= #TCQ CAL1_PB_DETECT_EDGE;
2762 
2763  // Look for an edge on all DQ bits in current DQS group
2764  CAL1_PB_DETECT_EDGE:
2765  if (detect_edge_done_r) begin
2766  if (found_stable_eye_r) begin
2767  // If we've found the left edge for all bits (or more precisely,
2768  // we've found the left edge, and then part of the stable
2769  // window thereafter), then proceed to positioning the CPT clock
2770  // right before the left margin
2771  cnt_idel_dec_cpt_r <= #TCQ MIN_EYE_SIZE + 1;
2772  cal1_state_r <= #TCQ CAL1_PB_DEC_CPT_LEFT;
2773  end else begin
2774  // If we've reached the end of the sampling time, and haven't
2775  // yet found the left margin of all the DQ bits, then:
2776  if (!tap_limit_cpt_r) begin
2777  // If we still have taps left to use, then store current value
2778  // of read data, increment the capture clock, and continue to
2779  // look for (left) edges
2780  store_sr_req_r <= #TCQ 1'b1;
2781  cal1_state_r <= #TCQ CAL1_PB_INC_CPT;
2782  end else begin
2783  // If we ran out of taps moving the capture clock, and we
2784  // haven't finished edge detection, then reset the capture
2785  // clock taps to 0 (gradually, one tap at a time...
2786  // then exit the per-bit portion of the algorithm -
2787  // i.e. proceed to adjust the capture clock and DQ IODELAYs as
2788  cnt_idel_dec_cpt_r <= #TCQ 6'd63;
2789  cal1_state_r <= #TCQ CAL1_PB_DEC_CPT;
2790  end
2791  end
2792  end
2793 
2794  // Increment delay for DQS
2795  CAL1_PB_INC_CPT: begin
2796  cal1_dlyce_cpt_r <= #TCQ 1'b1;
2797  cal1_dlyinc_cpt_r <= #TCQ 1'b1;
2798  cal1_state_r <= #TCQ CAL1_PB_INC_CPT_WAIT;
2799  end
2800 
2801  // Wait for IODELAY for both capture and internal nodes within
2802  // ISERDES to settle, before checking again for an edge
2803  CAL1_PB_INC_CPT_WAIT: begin
2804  cal1_dlyce_cpt_r <= #TCQ 1'b0;
2805  cal1_dlyinc_cpt_r <= #TCQ 1'b0;
2806  if (!cal1_wait_r)
2807  cal1_state_r <= #TCQ CAL1_PB_DETECT_EDGE;
2808  end
2809  // We've found the left edges of the windows for all DQ bits
2810  // (actually, we found it MIN_EYE_SIZE taps ago) Decrement capture
2811  // clock IDELAY to position just outside left edge of data window
2812  CAL1_PB_DEC_CPT_LEFT:
2813  if (cnt_idel_dec_cpt_r == 6'b000000)
2814  cal1_state_r <= #TCQ CAL1_PB_DEC_CPT_LEFT_WAIT;
2815  else begin
2816  cal1_dlyce_cpt_r <= #TCQ 1'b1;
2817  cal1_dlyinc_cpt_r <= #TCQ 1'b0;
2818  cnt_idel_dec_cpt_r <= #TCQ cnt_idel_dec_cpt_r - 1;
2819  end
2820 
2821  CAL1_PB_DEC_CPT_LEFT_WAIT:
2822  if (!cal1_wait_r)
2823  cal1_state_r <= #TCQ CAL1_PB_DETECT_EDGE_DQ;
2824 
2825  // If there is skew between individual DQ bits, then after we've
2826  // positioned the CPT clock, we will be "in the window" for some
2827  // DQ bits ("early" DQ bits), and "out of the window" for others
2828  // ("late" DQ bits). Increase DQ taps until we are out of the
2829  // window for all DQ bits
2830  CAL1_PB_DETECT_EDGE_DQ:
2831  if (detect_edge_done_r)
2832  if (found_edge_all_r) begin
2833  // We're out of the window for all DQ bits in this DQS group
2834  // We're done with per-bit deskew for this group - now decr
2835  // capture clock IODELAY tap count back to 0, and proceed
2836  // with the rest of stage 1 calibration for this DQS group
2837  cnt_idel_dec_cpt_r <= #TCQ tap_cnt_cpt_r;
2838  cal1_state_r <= #TCQ CAL1_PB_DEC_CPT;
2839  end else
2840  if (!idel_tap_limit_dq_pb_r)
2841  // If we still have DQ taps available for deskew, keep
2842  // incrementing IODELAY tap count for the appropriate DQ bits
2843  cal1_state_r <= #TCQ CAL1_PB_INC_DQ;
2844  else begin
2845  // Otherwise, stop immediately (we've done the best we can)
2846  // and proceed with rest of stage 1 calibration
2847  cnt_idel_dec_cpt_r <= #TCQ tap_cnt_cpt_r;
2848  cal1_state_r <= #TCQ CAL1_PB_DEC_CPT;
2849  end
2850 
2851  CAL1_PB_INC_DQ: begin
2852  // Increment only those DQ for which an edge hasn't been found yet
2853  cal1_dlyce_dq_r <= #TCQ ~pb_found_edge_last_r;
2854  cal1_dlyinc_dq_r <= #TCQ 1'b1;
2855  cal1_state_r <= #TCQ CAL1_PB_INC_DQ_WAIT;
2856  end
2857 
2858  CAL1_PB_INC_DQ_WAIT:
2859  if (!cal1_wait_r)
2860  cal1_state_r <= #TCQ CAL1_PB_DETECT_EDGE_DQ;
2861 
2862  // Decrement capture clock taps back to initial value
2863  CAL1_PB_DEC_CPT:
2864  if (cnt_idel_dec_cpt_r == 6'b000000)
2865  cal1_state_r <= #TCQ CAL1_PB_DEC_CPT_WAIT;
2866  else begin
2867  cal1_dlyce_cpt_r <= #TCQ 1'b1;
2868  cal1_dlyinc_cpt_r <= #TCQ 1'b0;
2869  cnt_idel_dec_cpt_r <= #TCQ cnt_idel_dec_cpt_r - 1;
2870  end
2871 
2872  // Wait for capture clock to settle, then proceed to rest of
2873  // state 1 calibration for this DQS group
2874  CAL1_PB_DEC_CPT_WAIT:
2875  if (!cal1_wait_r) begin
2876  store_sr_req_r <= #TCQ 1'b1;
2877  cal1_state_r <= #TCQ CAL1_STORE_FIRST_WAIT;
2878  end
2879 
2880  // When first starting calibration for a DQS group, save the
2881  // current value of the read data shift register, and use this
2882  // as a reference. Note that for the first iteration of the
2883  // edge detection loop, we will in effect be checking for an edge
2884  // at IODELAY taps = 0 - normally, we are comparing the read data
2885  // for IODELAY taps = N, with the read data for IODELAY taps = N-1
2886  // An edge can only be found at IODELAY taps = 0 if the read data
2887  // is changing during this time (possible due to jitter)
2888  CAL1_STORE_FIRST_WAIT: begin
2889  mpr_dec_cpt_r <= #TCQ 1'b0;
2890  if (!cal1_wait_r)
2891  cal1_state_r <= #TCQ CAL1_PAT_DETECT;
2892  end
2893 
2894  CAL1_VALID_WAIT: begin
2895  if (!cal1_wait_r)
2896  cal1_state_r <= #TCQ CAL1_MPR_PAT_DETECT;
2897  end
2898 
2899  CAL1_MPR_PAT_DETECT: begin
2900  // MPR read leveling for centering DQS in valid window before
2901  // OCLKDELAYED calibration begins in order to eliminate read issues
2902  if (idel_pat_detect_valid_r == 1'b0) begin
2903  cal1_state_r <= #TCQ CAL1_VALID_WAIT;
2904  idel_pat_detect_valid_r <= #TCQ 1'b1;
2905  end else if (idel_pat_detect_valid_r && idel_mpr_pat_detect_r) begin
2906  cal1_state_r <= #TCQ CAL1_DETECT_EDGE;
2907  idel_dec_cnt <= #TCQ 'd0;
2908  end else if (!idelay_tap_limit_r)
2909  cal1_state_r <= #TCQ CAL1_DQ_IDEL_TAP_INC;
2910  else
2911  cal1_state_r <= #TCQ CAL1_RDLVL_ERR;
2912  end
2913 
2914  CAL1_PAT_DETECT: begin
2915  // All DQ bits associated with a DQS are pushed to the right one IDELAY
2916  // tap at a time until first rising DQS is in the tri-state region
2917  // before first rising edge window.
2918  // The detect_edge_done_r condition included to support averaging
2919  // during IDELAY tap increments
2920  if (detect_edge_done_r) begin
2921  if (idel_pat_data_match) begin
2922  cal1_state_r <= #TCQ CAL1_DETECT_EDGE;
2923  idel_dec_cnt <= #TCQ 'd0;
2924  end else if (!idelay_tap_limit_r) begin
2925  cal1_state_r <= #TCQ CAL1_DQ_IDEL_TAP_INC;
2926  end else begin
2927  cal1_state_r <= #TCQ CAL1_RDLVL_ERR;
2928  end
2929  end
2930  end
2931 
2932  // Increment IDELAY tap by 1 for DQ bits in the byte being calibrated
2933  // until left edge of valid window detected
2934  CAL1_DQ_IDEL_TAP_INC: begin
2935  cal1_dq_idel_ce <= #TCQ 1'b1;
2936  cal1_dq_idel_inc <= #TCQ 1'b1;
2937  cal1_state_r <= #TCQ CAL1_DQ_IDEL_TAP_INC_WAIT;
2938  idel_pat_detect_valid_r <= #TCQ 1'b0;
2939  end
2940 
2941  CAL1_DQ_IDEL_TAP_INC_WAIT: begin
2942  cal1_dq_idel_ce <= #TCQ 1'b0;
2943  cal1_dq_idel_inc <= #TCQ 1'b0;
2944  if (!cal1_wait_r) begin
2945  if (~mpr_rdlvl_done_r & (DRAM_TYPE == "DDR3"))
2946  cal1_state_r <= #TCQ CAL1_MPR_PAT_DETECT;
2947  else
2948  cal1_state_r <= #TCQ CAL1_PAT_DETECT;
2949  end
2950  end
2951 
2952  // Decrement by 2 IDELAY taps once idel_pat_data_match detected
2953  CAL1_DQ_IDEL_TAP_DEC: begin
2954  cal1_dq_idel_inc <= #TCQ 1'b0;
2955  cal1_state_r <= #TCQ CAL1_DQ_IDEL_TAP_DEC_WAIT;
2956  if (idel_dec_cnt >= 'd0)
2957  cal1_dq_idel_ce <= #TCQ 1'b1;
2958  else
2959  cal1_dq_idel_ce <= #TCQ 1'b0;
2960  if (idel_dec_cnt > 'd0)
2961  idel_dec_cnt <= #TCQ idel_dec_cnt - 1;
2962  else
2963  idel_dec_cnt <= #TCQ idel_dec_cnt;
2964  end
2965 
2966  CAL1_DQ_IDEL_TAP_DEC_WAIT: begin
2967  cal1_dq_idel_ce <= #TCQ 1'b0;
2968  cal1_dq_idel_inc <= #TCQ 1'b0;
2969  if (!cal1_wait_r) begin
2970  if ((idel_dec_cnt > 'd0) || (pi_rdval_cnt > 'd0))
2971  cal1_state_r <= #TCQ CAL1_DQ_IDEL_TAP_DEC;
2972  else if (mpr_dec_cpt_r)
2973  cal1_state_r <= #TCQ CAL1_STORE_FIRST_WAIT;
2974  else
2975  cal1_state_r <= #TCQ CAL1_DETECT_EDGE;
2976  end
2977  end
2978 
2979  // Check for presence of data eye edge. During this state, we
2980  // sample the read data multiple times, and look for changes
2981  // in the read data, specifically:
2982  // 1. A change in the read data compared with the value of
2983  // read data from the previous delay tap. This indicates
2984  // that the most recent tap delay increment has moved us
2985  // into either a new window, or moved/kept us in the
2986  // transition/jitter region between windows. Note that this
2987  // condition only needs to be checked for once, and for
2988  // logistical purposes, we check this soon after entering
2989  // this state (see comment in CAL1_DETECT_EDGE below for
2990  // why this is done)
2991  // 2. A change in the read data while we are in this state
2992  // (i.e. in the absence of a tap delay increment). This
2993  // indicates that we're close enough to a window edge that
2994  // jitter will cause the read data to change even in the
2995  // absence of a tap delay change
2996  CAL1_DETECT_EDGE: begin
2997  // Essentially wait for the first comparision to finish, then
2998  // store current data into "old" data register. This store
2999  // happens now, rather than later (e.g. when we've have already
3000  // left this state) in order to avoid the situation the data that
3001  // is stored as "old" data has not been used in an "active
3002  // comparison" - i.e. data is stored after the last comparison
3003  // of this state. In this case, we can miss an edge if the
3004  // following sequence occurs:
3005  // 1. Comparison completes in this state - no edge found
3006  // 2. "Momentary jitter" occurs which "pushes" the data out the
3007  // equivalent of one delay tap
3008  // 3. We store this jittered data as the "old" data
3009  // 4. "Jitter" no longer present
3010  // 5. We increment the delay tap by one
3011  // 6. Now we compare the current with the "old" data - they're
3012  // the same, and no edge is detected
3013  // NOTE: Given the large # of comparisons done in this state, it's
3014  // highly unlikely the above sequence will occur in actual H/W
3015 
3016  // Wait for the first load of read data into the comparison
3017  // shift register to finish, then load the current read data
3018  // into the "old" data register. This allows us to do one
3019  // initial comparision between the current read data, and
3020  // stored data corresponding to the previous delay tap
3021  idel_pat_detect_valid_r <= #TCQ 1'b0;
3022  if (!store_sr_req_pulsed_r) begin
3023  // Pulse store_sr_req_r only once in this state
3024  store_sr_req_r <= #TCQ 1'b1;
3025  store_sr_req_pulsed_r <= #TCQ 1'b1;
3026  end else begin
3027  store_sr_req_r <= #TCQ 1'b0;
3028  store_sr_req_pulsed_r <= #TCQ 1'b1;
3029  end
3030 
3031  // Continue to sample read data and look for edges until the
3032  // appropriate time interval (shorter for simulation-only,
3033  // much, much longer for actual h/w) has elapsed
3034  if (detect_edge_done_r) begin
3035  if (tap_limit_cpt_r)
3036  // Only one edge detected and ran out of taps since only one
3037  // bit time worth of taps available for window detection. This
3038  // can happen if at tap 0 DQS is in previous window which results
3039  // in only left edge being detected. Or at tap 0 DQS is in the
3040  // current window resulting in only right edge being detected.
3041  // Depending on the frequency this case can also happen if at
3042  // tap 0 DQS is in the left noise region resulting in only left
3043  // edge being detected.
3044  cal1_state_r <= #TCQ CAL1_CALC_IDEL;
3045  else if (found_edge_r) begin
3046  // Sticky bit - asserted after we encounter an edge, although
3047  // the current edge may not be considered the "first edge" this
3048  // just means we found at least one edge
3049  found_first_edge_r <= #TCQ 1'b1;
3050 
3051  // Only the right edge of the data valid window is found
3052  // Record the inner right edge tap value
3053  if (!found_first_edge_r && found_stable_eye_last_r) begin
3054  if (tap_cnt_cpt_r == 'd0)
3055  right_edge_taps_r <= #TCQ 'd0;
3056  else
3057  right_edge_taps_r <= #TCQ tap_cnt_cpt_r;
3058  end
3059 
3060  // Both edges of data valid window found:
3061  // If we've found a second edge after a region of stability
3062  // then we must have just passed the second ("right" edge of
3063  // the window. Record this second_edge_taps = current tap-1,
3064  // because we're one past the actual second edge tap, where
3065  // the edge taps represent the extremes of the data valid
3066  // window (i.e. smallest & largest taps where data still valid
3067  if (found_first_edge_r && found_stable_eye_last_r) begin
3068  found_second_edge_r <= #TCQ 1'b1;
3069  second_edge_taps_r <= #TCQ tap_cnt_cpt_r - 1;
3070  cal1_state_r <= #TCQ CAL1_CALC_IDEL;
3071  end else begin
3072  // Otherwise, an edge was found (just not the "second" edge)
3073  // Assuming DQS is in the correct window at tap 0 of Phaser IN
3074  // fine tap. The first edge found is the right edge of the valid
3075  // window and is the beginning of the jitter region hence done!
3076  first_edge_taps_r <= #TCQ tap_cnt_cpt_r;
3077  cal1_state_r <= #TCQ CAL1_IDEL_INC_CPT;
3078  end
3079  end else
3080  // Otherwise, if we haven't found an edge....
3081  // If we still have taps left to use, then keep incrementing
3082  cal1_state_r <= #TCQ CAL1_IDEL_INC_CPT;
3083  end
3084  end
3085 
3086  // Increment Phaser_IN delay for DQS
3087  CAL1_IDEL_INC_CPT: begin
3088  cal1_state_r <= #TCQ CAL1_IDEL_INC_CPT_WAIT;
3089  if (~tap_limit_cpt_r) begin
3090  cal1_dlyce_cpt_r <= #TCQ 1'b1;
3091  cal1_dlyinc_cpt_r <= #TCQ 1'b1;
3092  end else begin
3093  cal1_dlyce_cpt_r <= #TCQ 1'b0;
3094  cal1_dlyinc_cpt_r <= #TCQ 1'b0;
3095  end
3096  end
3097 
3098  // Wait for Phaser_In to settle, before checking again for an edge
3099  CAL1_IDEL_INC_CPT_WAIT: begin
3100  cal1_dlyce_cpt_r <= #TCQ 1'b0;
3101  cal1_dlyinc_cpt_r <= #TCQ 1'b0;
3102  if (!cal1_wait_r)
3103  cal1_state_r <= #TCQ CAL1_DETECT_EDGE;
3104  end
3105 
3106  // Calculate final value of Phaser_IN taps. At this point, one or both
3107  // edges of data eye have been found, and/or all taps have been
3108  // exhausted looking for the edges
3109  // NOTE: We're calculating the amount to decrement by, not the
3110  // absolute setting for DQS.
3111  CAL1_CALC_IDEL: begin
3112  // CASE1: If 2 edges found.
3113  if (found_second_edge_r)
3114  cnt_idel_dec_cpt_r
3115  <= #TCQ ((second_edge_taps_r -
3116  first_edge_taps_r)>>1) + 1;
3117  else if (right_edge_taps_r > 6'd0)
3118  // Only right edge detected
3119  // right_edge_taps_r is the inner right edge tap value
3120  // hence used for calculation
3121  cnt_idel_dec_cpt_r
3122  <= #TCQ (tap_cnt_cpt_r - (right_edge_taps_r>>1));
3123  else if (found_first_edge_r)
3124  // Only left edge detected
3125  cnt_idel_dec_cpt_r
3126  <= #TCQ ((tap_cnt_cpt_r - first_edge_taps_r)>>1);
3127  else
3128  cnt_idel_dec_cpt_r
3129  <= #TCQ (tap_cnt_cpt_r>>1);
3130  // Now use the value we just calculated to decrement CPT taps
3131  // to the desired calibration point
3132  cal1_state_r <= #TCQ CAL1_IDEL_DEC_CPT;
3133  end
3134 
3135  // decrement capture clock for final adjustment - center
3136  // capture clock in middle of data eye. This adjustment will occur
3137  // only when both the edges are found usign CPT taps. Must do this
3138  // incrementally to avoid clock glitching (since CPT drives clock
3139  // divider within each ISERDES)
3140  CAL1_IDEL_DEC_CPT: begin
3141  cal1_dlyce_cpt_r <= #TCQ 1'b1;
3142  cal1_dlyinc_cpt_r <= #TCQ 1'b0;
3143  // once adjustment is complete, we're done with calibration for
3144  // this DQS, repeat for next DQS
3145  cnt_idel_dec_cpt_r <= #TCQ cnt_idel_dec_cpt_r - 1;
3146  if (cnt_idel_dec_cpt_r == 6'b000001) begin
3147  if (mpr_dec_cpt_r) begin
3148  if (|idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing]) begin
3149  idel_dec_cnt <= #TCQ idelay_tap_cnt_r[rnk_cnt_r][cal1_cnt_cpt_timing];
3150  cal1_state_r <= #TCQ CAL1_DQ_IDEL_TAP_DEC;
3151  end else
3152  cal1_state_r <= #TCQ CAL1_STORE_FIRST_WAIT;
3153  end else
3154  cal1_state_r <= #TCQ CAL1_NEXT_DQS;
3155  end else
3156  cal1_state_r <= #TCQ CAL1_IDEL_DEC_CPT_WAIT;
3157  end
3158 
3159  CAL1_IDEL_DEC_CPT_WAIT: begin
3160  cal1_dlyce_cpt_r <= #TCQ 1'b0;
3161  cal1_dlyinc_cpt_r <= #TCQ 1'b0;
3162  if (!cal1_wait_r)
3163  cal1_state_r <= #TCQ CAL1_IDEL_DEC_CPT;
3164  end
3165 
3166  // Determine whether we're done, or have more DQS's to calibrate
3167  // Also request precharge after every byte, as appropriate
3168  CAL1_NEXT_DQS: begin
3169  //if (mpr_rdlvl_done_r || (DRAM_TYPE == "DDR2"))
3170  cal1_prech_req_r <= #TCQ 1'b1;
3171  //else
3172  // cal1_prech_req_r <= #TCQ 1'b0;
3173  cal1_dlyce_cpt_r <= #TCQ 1'b0;
3174  cal1_dlyinc_cpt_r <= #TCQ 1'b0;
3175  // Prepare for another iteration with next DQS group
3176  found_first_edge_r <= #TCQ 1'b0;
3177  found_second_edge_r <= #TCQ 1'b0;
3178  first_edge_taps_r <= #TCQ 'd0;
3179  second_edge_taps_r <= #TCQ 'd0;
3180  if ((SIM_CAL_OPTION == "FAST_CAL") ||
3181  (cal1_cnt_cpt_r >= DQS_WIDTH-1)) begin
3182  if (mpr_rdlvl_done_r) begin
3183  rdlvl_last_byte_done <= #TCQ 1'b1;
3184  mpr_last_byte_done <= #TCQ 1'b0;
3185  end else begin
3186  rdlvl_last_byte_done <= #TCQ 1'b0;
3187  mpr_last_byte_done <= #TCQ 1'b1;
3188  end
3189  end
3190 
3191  // Wait until precharge that occurs in between calibration of
3192  // DQS groups is finished
3193  if (prech_done) begin // || (~mpr_rdlvl_done_r & (DRAM_TYPE == "DDR3"))) begin
3194  if (SIM_CAL_OPTION == "FAST_CAL") begin
3195  //rdlvl_rank_done_r <= #TCQ 1'b1;
3196  rdlvl_last_byte_done <= #TCQ 1'b0;
3197  mpr_last_byte_done <= #TCQ 1'b0;
3198  cal1_state_r <= #TCQ CAL1_DONE; //CAL1_REGL_LOAD;
3199  end else if (cal1_cnt_cpt_r >= DQS_WIDTH-1) begin
3200  if (~mpr_rdlvl_done_r) begin
3201  mpr_rank_done_r <= #TCQ 1'b1;
3202  // if (rnk_cnt_r == RANKS-1) begin
3203  // All DQS groups in all ranks done
3204  cal1_state_r <= #TCQ CAL1_DONE;
3205  cal1_cnt_cpt_r <= #TCQ 'b0;
3206  // end else begin
3207  // // Process DQS groups in next rank
3208  // rnk_cnt_r <= #TCQ rnk_cnt_r + 1;
3209  // new_cnt_cpt_r <= #TCQ 1'b1;
3210  // cal1_cnt_cpt_r <= #TCQ 'b0;
3211  // cal1_state_r <= #TCQ CAL1_IDLE;
3212  // end
3213  end else begin
3214  // All DQS groups in a rank done
3215  rdlvl_rank_done_r <= #TCQ 1'b1;
3216  if (rnk_cnt_r == RANKS-1) begin
3217  // All DQS groups in all ranks done
3218  cal1_state_r <= #TCQ CAL1_REGL_LOAD;
3219  end else begin
3220  // Process DQS groups in next rank
3221  rnk_cnt_r <= #TCQ rnk_cnt_r + 1;
3222  new_cnt_cpt_r <= #TCQ 1'b1;
3223  cal1_cnt_cpt_r <= #TCQ 'b0;
3224  cal1_state_r <= #TCQ CAL1_IDLE;
3225  end
3226  end
3227  end else begin
3228  // Process next DQS group
3229  new_cnt_cpt_r <= #TCQ 1'b1;
3230  cal1_cnt_cpt_r <= #TCQ cal1_cnt_cpt_r + 1;
3231  cal1_state_r <= #TCQ CAL1_NEW_DQS_PREWAIT;
3232  end
3233  end
3234  end
3235 
3236  CAL1_NEW_DQS_PREWAIT: begin
3237  if (!cal1_wait_r) begin
3238  if (~mpr_rdlvl_done_r & (DRAM_TYPE == "DDR3"))
3239  cal1_state_r <= #TCQ CAL1_MPR_NEW_DQS_WAIT;
3240  else
3241  cal1_state_r <= #TCQ CAL1_NEW_DQS_WAIT;
3242  end
3243  end
3244 
3245  // Load rank registers in Phaser_IN
3246  CAL1_REGL_LOAD: begin
3247  rdlvl_rank_done_r <= #TCQ 1'b0;
3248  mpr_rank_done_r <= #TCQ 1'b0;
3249  cal1_prech_req_r <= #TCQ 1'b0;
3250  cal1_cnt_cpt_r <= #TCQ 'b0;
3251  rnk_cnt_r <= #TCQ 2'b00;
3252  if ((regl_rank_cnt == RANKS-1) &&
3253  ((regl_dqs_cnt == DQS_WIDTH-1) && (done_cnt == 4'd1))) begin
3254  cal1_state_r <= #TCQ CAL1_DONE;
3255  rdlvl_last_byte_done <= #TCQ 1'b0;
3256  mpr_last_byte_done <= #TCQ 1'b0;
3257  end else
3258  cal1_state_r <= #TCQ CAL1_REGL_LOAD;
3259  end
3260 
3261  CAL1_RDLVL_ERR: begin
3262  rdlvl_stg1_err <= #TCQ 1'b1;
3263  end
3264 
3265  // Done with this stage of calibration
3266  // if used, allow DEBUG_PORT to control taps
3267  CAL1_DONE: begin
3268  mpr_rdlvl_done_r <= #TCQ 1'b1;
3269  cal1_prech_req_r <= #TCQ 1'b0;
3270  if (~mpr_rdlvl_done_r && (OCAL_EN=="ON") && (DRAM_TYPE == "DDR3")) begin
3271  rdlvl_stg1_done <= #TCQ 1'b0;
3272  cal1_state_r <= #TCQ CAL1_IDLE;
3273  end else
3274  rdlvl_stg1_done <= #TCQ 1'b1;
3275  end
3276 
3277  endcase
3278  end
3279 
3280 
3281 
3282 
3283 
3284 endmodule
Generated on Wed Apr 18 2018 10:55:27 for AMC13 by   doxygen 1.8.1